LLVM  7.0.0svn
SLPVectorizer.cpp
Go to the documentation of this file.
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11 // stores that can be put together into vector-stores. Next, it attempts to
12 // construct vectorizable tree using the use-def chains. If a profitable tree
13 // was found, the SLP vectorizer performs vectorization on the tree.
14 //
15 // The pass is inspired by the work described in the paper:
16 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17 //
18 //===----------------------------------------------------------------------===//
19 
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/DenseSet.h"
24 #include "llvm/ADT/MapVector.h"
25 #include "llvm/ADT/None.h"
26 #include "llvm/ADT/Optional.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SetVector.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/SmallSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/Statistic.h"
34 #include "llvm/ADT/iterator.h"
41 #include "llvm/Analysis/LoopInfo.h"
50 #include "llvm/IR/Attributes.h"
51 #include "llvm/IR/BasicBlock.h"
52 #include "llvm/IR/Constant.h"
53 #include "llvm/IR/Constants.h"
54 #include "llvm/IR/DataLayout.h"
55 #include "llvm/IR/DebugLoc.h"
56 #include "llvm/IR/DerivedTypes.h"
57 #include "llvm/IR/Dominators.h"
58 #include "llvm/IR/Function.h"
59 #include "llvm/IR/IRBuilder.h"
60 #include "llvm/IR/InstrTypes.h"
61 #include "llvm/IR/Instruction.h"
62 #include "llvm/IR/Instructions.h"
63 #include "llvm/IR/IntrinsicInst.h"
64 #include "llvm/IR/Intrinsics.h"
65 #include "llvm/IR/Module.h"
66 #include "llvm/IR/NoFolder.h"
67 #include "llvm/IR/Operator.h"
68 #include "llvm/IR/PassManager.h"
69 #include "llvm/IR/PatternMatch.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/Pass.h"
77 #include "llvm/Support/Casting.h"
79 #include "llvm/Support/Compiler.h"
81 #include "llvm/Support/Debug.h"
84 #include "llvm/Support/KnownBits.h"
89 #include <algorithm>
90 #include <cassert>
91 #include <cstdint>
92 #include <iterator>
93 #include <memory>
94 #include <set>
95 #include <string>
96 #include <tuple>
97 #include <utility>
98 #include <vector>
99 
100 using namespace llvm;
101 using namespace llvm::PatternMatch;
102 using namespace slpvectorizer;
103 
104 #define SV_NAME "slp-vectorizer"
105 #define DEBUG_TYPE "SLP"
106 
107 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
108 
109 static cl::opt<int>
110  SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111  cl::desc("Only vectorize if you gain more than this "
112  "number "));
113 
114 static cl::opt<bool>
115 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116  cl::desc("Attempt to vectorize horizontal reductions"));
117 
119  "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120  cl::desc(
121  "Attempt to vectorize horizontal reductions feeding into a store"));
122 
123 static cl::opt<int>
124 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125  cl::desc("Attempt to vectorize for this register size in bits"));
126 
127 /// Limits the size of scheduling regions in a block.
128 /// It avoid long compile times for _very_ large blocks where vector
129 /// instructions are spread over a wide range.
130 /// This limit is way higher than needed by real-world functions.
131 static cl::opt<int>
132 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133  cl::desc("Limit the size of the SLP scheduling region per block"));
134 
136  "slp-min-reg-size", cl::init(128), cl::Hidden,
137  cl::desc("Attempt to vectorize for this register size in bits"));
138 
140  "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141  cl::desc("Limit the recursion depth when building a vectorizable tree"));
142 
144  "slp-min-tree-size", cl::init(3), cl::Hidden,
145  cl::desc("Only vectorize small trees if they are fully vectorizable"));
146 
147 static cl::opt<bool>
148  ViewSLPTree("view-slp-tree", cl::Hidden,
149  cl::desc("Display the SLP trees with Graphviz"));
150 
151 // Limit the number of alias checks. The limit is chosen so that
152 // it has no negative effect on the llvm benchmarks.
153 static const unsigned AliasedCheckLimit = 10;
154 
155 // Another limit for the alias checks: The maximum distance between load/store
156 // instructions where alias checks are done.
157 // This limit is useful for very large basic blocks.
158 static const unsigned MaxMemDepDistance = 160;
159 
160 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161 /// regions to be handled.
162 static const int MinScheduleRegionSize = 16;
163 
164 /// \brief Predicate for the element types that the SLP vectorizer supports.
165 ///
166 /// The most important thing to filter here are types which are invalid in LLVM
167 /// vectors. We also filter target specific types which have absolutely no
168 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169 /// avoids spending time checking the cost model and realizing that they will
170 /// be inevitably scalarized.
171 static bool isValidElementType(Type *Ty) {
172  return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173  !Ty->isPPC_FP128Ty();
174 }
175 
176 /// \returns true if all of the instructions in \p VL are in the same block or
177 /// false otherwise.
179  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180  if (!I0)
181  return false;
182  BasicBlock *BB = I0->getParent();
183  for (int i = 1, e = VL.size(); i < e; i++) {
184  Instruction *I = dyn_cast<Instruction>(VL[i]);
185  if (!I)
186  return false;
187 
188  if (BB != I->getParent())
189  return false;
190  }
191  return true;
192 }
193 
194 /// \returns True if all of the values in \p VL are constants.
196  for (Value *i : VL)
197  if (!isa<Constant>(i))
198  return false;
199  return true;
200 }
201 
202 /// \returns True if all of the values in \p VL are identical.
203 static bool isSplat(ArrayRef<Value *> VL) {
204  for (unsigned i = 1, e = VL.size(); i < e; ++i)
205  if (VL[i] != VL[0])
206  return false;
207  return true;
208 }
209 
210 /// Checks if the vector of instructions can be represented as a shuffle, like:
211 /// %x0 = extractelement <4 x i8> %x, i32 0
212 /// %x3 = extractelement <4 x i8> %x, i32 3
213 /// %y1 = extractelement <4 x i8> %y, i32 1
214 /// %y2 = extractelement <4 x i8> %y, i32 2
215 /// %x0x0 = mul i8 %x0, %x0
216 /// %x3x3 = mul i8 %x3, %x3
217 /// %y1y1 = mul i8 %y1, %y1
218 /// %y2y2 = mul i8 %y2, %y2
219 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223 /// ret <4 x i8> %ins4
224 /// can be transformed into:
225 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226 /// i32 6>
227 /// %2 = mul <4 x i8> %1, %1
228 /// ret <4 x i8> %2
229 /// We convert this initially to something like:
230 /// %x0 = extractelement <4 x i8> %x, i32 0
231 /// %x3 = extractelement <4 x i8> %x, i32 3
232 /// %y1 = extractelement <4 x i8> %y, i32 1
233 /// %y2 = extractelement <4 x i8> %y, i32 2
234 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238 /// %5 = mul <4 x i8> %4, %4
239 /// %6 = extractelement <4 x i8> %5, i32 0
240 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241 /// %7 = extractelement <4 x i8> %5, i32 1
242 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243 /// %8 = extractelement <4 x i8> %5, i32 2
244 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245 /// %9 = extractelement <4 x i8> %5, i32 3
246 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247 /// ret <4 x i8> %ins4
248 /// InstCombiner transforms this into a shuffle and vector mul
251  auto *EI0 = cast<ExtractElementInst>(VL[0]);
252  unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253  Value *Vec1 = nullptr;
254  Value *Vec2 = nullptr;
255  enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256  ShuffleMode CommonShuffleMode = Unknown;
257  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258  auto *EI = cast<ExtractElementInst>(VL[I]);
259  auto *Vec = EI->getVectorOperand();
260  // All vector operands must have the same number of vector elements.
261  if (Vec->getType()->getVectorNumElements() != Size)
262  return None;
263  auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264  if (!Idx)
265  return None;
266  // Undefined behavior if Idx is negative or >= Size.
267  if (Idx->getValue().uge(Size))
268  continue;
269  unsigned IntIdx = Idx->getValue().getZExtValue();
270  // We can extractelement from undef vector.
271  if (isa<UndefValue>(Vec))
272  continue;
273  // For correct shuffling we have to have at most 2 different vector operands
274  // in all extractelement instructions.
275  if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276  return None;
277  if (CommonShuffleMode == Permute)
278  continue;
279  // If the extract index is not the same as the operation number, it is a
280  // permutation.
281  if (IntIdx != I) {
282  CommonShuffleMode = Permute;
283  continue;
284  }
285  // Check the shuffle mode for the current operation.
286  if (!Vec1)
287  Vec1 = Vec;
288  else if (Vec != Vec1)
289  Vec2 = Vec;
290  // Example: shufflevector A, B, <0,5,2,7>
291  // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292  // I is even and IntIdx for B == I - FirstAlternate shuffle.
293  // Example: shufflevector A, B, <4,1,6,3>
294  // I is even and IntIdx for A == I - SecondAlternate shuffle.
295  // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296  const bool IIsEven = I & 1;
297  const bool CurrVecIsA = Vec == Vec1;
298  const bool IIsOdd = !IIsEven;
299  const bool CurrVecIsB = !CurrVecIsA;
300  ShuffleMode CurrentShuffleMode =
301  ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302  : SecondAlternate;
303  // Common mode is not set or the same as the shuffle mode of the current
304  // operation - alternate.
305  if (CommonShuffleMode == Unknown)
306  CommonShuffleMode = CurrentShuffleMode;
307  // Common shuffle mode is not the same as the shuffle mode of the current
308  // operation - permutation.
309  if (CommonShuffleMode != CurrentShuffleMode)
310  CommonShuffleMode = Permute;
311  }
312  // If we're not crossing lanes in different vectors, consider it as blending.
313  if ((CommonShuffleMode == FirstAlternate ||
314  CommonShuffleMode == SecondAlternate) &&
315  Vec2)
317  // If Vec2 was never used, we have a permutation of a single vector, otherwise
318  // we have permutation of 2 vectors.
321 }
322 
323 ///\returns Opcode that can be clubbed with \p Op to create an alternate
324 /// sequence which can later be merged as a ShuffleVector instruction.
325 static unsigned getAltOpcode(unsigned Op) {
326  switch (Op) {
327  case Instruction::FAdd:
328  return Instruction::FSub;
329  case Instruction::FSub:
330  return Instruction::FAdd;
331  case Instruction::Add:
332  return Instruction::Sub;
333  case Instruction::Sub:
334  return Instruction::Add;
335  default:
336  return 0;
337  }
338 }
339 
340 static bool isOdd(unsigned Value) {
341  return Value & 1;
342 }
343 
344 static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345  unsigned CheckedOpcode) {
346  return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347 }
348 
349 /// Chooses the correct key for scheduling data. If \p Op has the same (or
350 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
351 /// OpValue.
352 static Value *isOneOf(Value *OpValue, Value *Op) {
353  auto *I = dyn_cast<Instruction>(Op);
354  if (!I)
355  return OpValue;
356  auto *OpInst = cast<Instruction>(OpValue);
357  unsigned OpInstOpcode = OpInst->getOpcode();
358  unsigned IOpcode = I->getOpcode();
359  if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
360  return Op;
361  return OpValue;
362 }
363 
364 namespace {
365 
366 /// Contains data for the instructions going to be vectorized.
367 struct RawInstructionsData {
368  /// Main Opcode of the instructions going to be vectorized.
369  unsigned Opcode = 0;
370 
371  /// The list of instructions have some instructions with alternate opcodes.
372  bool HasAltOpcodes = false;
373 };
374 
375 } // end anonymous namespace
376 
377 /// Checks the list of the vectorized instructions \p VL and returns info about
378 /// this list.
379 static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380  auto *I0 = dyn_cast<Instruction>(VL[0]);
381  if (!I0)
382  return {};
383  RawInstructionsData Res;
384  unsigned Opcode = I0->getOpcode();
385  // Walk through the list of the vectorized instructions
386  // in order to check its structure described by RawInstructionsData.
387  for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388  auto *I = dyn_cast<Instruction>(VL[Cnt]);
389  if (!I)
390  return {};
391  if (Opcode != I->getOpcode())
392  Res.HasAltOpcodes = true;
393  }
394  Res.Opcode = Opcode;
395  return Res;
396 }
397 
398 namespace {
399 
400 /// Main data required for vectorization of instructions.
401 struct InstructionsState {
402  /// The very first instruction in the list with the main opcode.
403  Value *OpValue = nullptr;
404 
405  /// The main opcode for the list of instructions.
406  unsigned Opcode = 0;
407 
408  /// Some of the instructions in the list have alternate opcodes.
409  bool IsAltShuffle = false;
410 
411  InstructionsState() = default;
412  InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413  : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
414 };
415 
416 } // end anonymous namespace
417 
418 /// \returns analysis of the Instructions in \p VL described in
419 /// InstructionsState, the Opcode that we suppose the whole list
420 /// could be vectorized even if its structure is diverse.
421 static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422  auto Res = getMainOpcode(VL);
423  unsigned Opcode = Res.Opcode;
424  if (!Res.HasAltOpcodes)
425  return InstructionsState(VL[0], Opcode, false);
426  auto *OpInst = cast<Instruction>(VL[0]);
427  unsigned AltOpcode = getAltOpcode(Opcode);
428  // Examine each element in the list instructions VL to determine
429  // if some operations there could be considered as an alternative
430  // (for example as subtraction relates to addition operation).
431  for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432  auto *I = cast<Instruction>(VL[Cnt]);
433  unsigned InstOpcode = I->getOpcode();
434  if ((Res.HasAltOpcodes &&
435  InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436  (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437  return InstructionsState(OpInst, 0, false);
438  }
439  }
440  return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
441 }
442 
443 /// \returns true if all of the values in \p VL have the same type or false
444 /// otherwise.
446  Type *Ty = VL[0]->getType();
447  for (int i = 1, e = VL.size(); i < e; i++)
448  if (VL[i]->getType() != Ty)
449  return false;
450 
451  return true;
452 }
453 
454 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
456  unsigned Opcode = E->getOpcode();
457  assert((Opcode == Instruction::ExtractElement ||
458  Opcode == Instruction::ExtractValue) &&
459  "Expected extractelement or extractvalue instruction.");
460  if (Opcode == Instruction::ExtractElement) {
461  auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
462  if (!CI)
463  return None;
464  return CI->getZExtValue();
465  }
466  ExtractValueInst *EI = cast<ExtractValueInst>(E);
467  if (EI->getNumIndices() != 1)
468  return None;
469  return *EI->idx_begin();
470 }
471 
472 /// \returns True if in-tree use also needs extract. This refers to
473 /// possible scalar operand in vectorized instruction.
475  TargetLibraryInfo *TLI) {
476  unsigned Opcode = UserInst->getOpcode();
477  switch (Opcode) {
478  case Instruction::Load: {
479  LoadInst *LI = cast<LoadInst>(UserInst);
480  return (LI->getPointerOperand() == Scalar);
481  }
482  case Instruction::Store: {
483  StoreInst *SI = cast<StoreInst>(UserInst);
484  return (SI->getPointerOperand() == Scalar);
485  }
486  case Instruction::Call: {
487  CallInst *CI = cast<CallInst>(UserInst);
489  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
490  return (CI->getArgOperand(1) == Scalar);
491  }
493  }
494  default:
495  return false;
496  }
497 }
498 
499 /// \returns the AA location that is being access by the instruction.
501  if (StoreInst *SI = dyn_cast<StoreInst>(I))
502  return MemoryLocation::get(SI);
503  if (LoadInst *LI = dyn_cast<LoadInst>(I))
504  return MemoryLocation::get(LI);
505  return MemoryLocation();
506 }
507 
508 /// \returns True if the instruction is not a volatile or atomic load/store.
509 static bool isSimple(Instruction *I) {
510  if (LoadInst *LI = dyn_cast<LoadInst>(I))
511  return LI->isSimple();
512  if (StoreInst *SI = dyn_cast<StoreInst>(I))
513  return SI->isSimple();
514  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
515  return !MI->isVolatile();
516  return true;
517 }
518 
519 namespace llvm {
520 
521 namespace slpvectorizer {
522 
523 /// Bottom Up SLP Vectorizer.
524 class BoUpSLP {
525 public:
532 
536  const DataLayout *DL, OptimizationRemarkEmitter *ORE)
537  : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
538  DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
539  CodeMetrics::collectEphemeralValues(F, AC, EphValues);
540  // Use the vector register size specified by the target unless overridden
541  // by a command-line option.
542  // TODO: It would be better to limit the vectorization factor based on
543  // data type rather than just register size. For example, x86 AVX has
544  // 256-bit registers, but it does not support integer operations
545  // at that width (that requires AVX2).
546  if (MaxVectorRegSizeOption.getNumOccurrences())
547  MaxVecRegSize = MaxVectorRegSizeOption;
548  else
549  MaxVecRegSize = TTI->getRegisterBitWidth(true);
550 
551  if (MinVectorRegSizeOption.getNumOccurrences())
552  MinVecRegSize = MinVectorRegSizeOption;
553  else
554  MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
555  }
556 
557  /// \brief Vectorize the tree that starts with the elements in \p VL.
558  /// Returns the vectorized root.
559  Value *vectorizeTree();
560 
561  /// Vectorize the tree but with the list of externally used values \p
562  /// ExternallyUsedValues. Values in this MapVector can be replaced but the
563  /// generated extractvalue instructions.
564  Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
565 
566  /// \returns the cost incurred by unwanted spills and fills, caused by
567  /// holding live values over call sites.
568  int getSpillCost();
569 
570  /// \returns the vectorization cost of the subtree that starts at \p VL.
571  /// A negative number means that this is profitable.
572  int getTreeCost();
573 
574  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575  /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
576  void buildTree(ArrayRef<Value *> Roots,
577  ArrayRef<Value *> UserIgnoreLst = None);
578 
579  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
580  /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
581  /// into account (anf updating it, if required) list of externally used
582  /// values stored in \p ExternallyUsedValues.
583  void buildTree(ArrayRef<Value *> Roots,
584  ExtraValueToDebugLocsMap &ExternallyUsedValues,
585  ArrayRef<Value *> UserIgnoreLst = None);
586 
587  /// Clear the internal data structures that are created by 'buildTree'.
588  void deleteTree() {
589  VectorizableTree.clear();
590  ScalarToTreeEntry.clear();
591  MustGather.clear();
592  ExternalUses.clear();
593  NumOpsWantToKeepOrder.clear();
594  NumOpsWantToKeepOriginalOrder = 0;
595  for (auto &Iter : BlocksSchedules) {
596  BlockScheduling *BS = Iter.second.get();
597  BS->clear();
598  }
599  MinBWs.clear();
600  }
601 
602  unsigned getTreeSize() const { return VectorizableTree.size(); }
603 
604  /// \brief Perform LICM and CSE on the newly generated gather sequences.
605  void optimizeGatherSequence();
606 
607  /// \returns The best order of instructions for vectorization.
609  auto I = std::max_element(
610  NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
611  [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
612  const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
613  return D1.second < D2.second;
614  });
615  if (I == NumOpsWantToKeepOrder.end() ||
616  I->getSecond() <= NumOpsWantToKeepOriginalOrder)
617  return None;
618 
619  return makeArrayRef(I->getFirst());
620  }
621 
622  /// \return The vector element size in bits to use when vectorizing the
623  /// expression tree ending at \p V. If V is a store, the size is the width of
624  /// the stored value. Otherwise, the size is the width of the largest loaded
625  /// value reaching V. This method is used by the vectorizer to calculate
626  /// vectorization factors.
627  unsigned getVectorElementSize(Value *V);
628 
629  /// Compute the minimum type sizes required to represent the entries in a
630  /// vectorizable tree.
632 
633  // \returns maximum vector register size as set by TTI or overridden by cl::opt.
634  unsigned getMaxVecRegSize() const {
635  return MaxVecRegSize;
636  }
637 
638  // \returns minimum vector register size as set by cl::opt.
639  unsigned getMinVecRegSize() const {
640  return MinVecRegSize;
641  }
642 
643  /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
644  ///
645  /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
646  unsigned canMapToVector(Type *T, const DataLayout &DL) const;
647 
648  /// \returns True if the VectorizableTree is both tiny and not fully
649  /// vectorizable. We do not vectorize such trees.
650  bool isTreeTinyAndNotFullyVectorizable();
651 
653 
654 private:
655  struct TreeEntry;
656 
657  /// Checks if all users of \p I are the part of the vectorization tree.
658  bool areAllUsersVectorized(Instruction *I) const;
659 
660  /// \returns the cost of the vectorizable entry.
661  int getEntryCost(TreeEntry *E);
662 
663  /// This is the recursive part of buildTree.
664  void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
665 
666  /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
667  /// be vectorized to use the original vector (or aggregate "bitcast" to a
668  /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
669  /// returns false, setting \p CurrentOrder to either an empty vector or a
670  /// non-identity permutation that allows to reuse extract instructions.
671  bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
672  SmallVectorImpl<unsigned> &CurrentOrder) const;
673 
674  /// Vectorize a single entry in the tree.
675  Value *vectorizeTree(TreeEntry *E);
676 
677  /// Vectorize a single entry in the tree, starting in \p VL.
678  Value *vectorizeTree(ArrayRef<Value *> VL);
679 
680  /// \returns the scalarization cost for this type. Scalarization in this
681  /// context means the creation of vectors from a group of scalars.
682  int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
683 
684  /// \returns the scalarization cost for this list of values. Assuming that
685  /// this subtree gets vectorized, we may need to extract the values from the
686  /// roots. This method calculates the cost of extracting the values.
687  int getGatherCost(ArrayRef<Value *> VL);
688 
689  /// \brief Set the Builder insert point to one after the last instruction in
690  /// the bundle
691  void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
692 
693  /// \returns a vector from a collection of scalars in \p VL.
694  Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
695 
696  /// \returns whether the VectorizableTree is fully vectorizable and will
697  /// be beneficial even the tree height is tiny.
698  bool isFullyVectorizableTinyTree();
699 
700  /// \reorder commutative operands in alt shuffle if they result in
701  /// vectorized code.
702  void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
705 
706  /// \reorder commutative operands to get better probability of
707  /// generating vectorized code.
708  void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
710  SmallVectorImpl<Value *> &Right);
711  struct TreeEntry {
712  TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
713 
714  /// \returns true if the scalars in VL are equal to this entry.
715  bool isSame(ArrayRef<Value *> VL) const {
716  if (VL.size() == Scalars.size())
717  return std::equal(VL.begin(), VL.end(), Scalars.begin());
718  return VL.size() == ReuseShuffleIndices.size() &&
719  std::equal(
720  VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
721  [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
722  }
723 
724  /// A vector of scalars.
725  ValueList Scalars;
726 
727  /// The Scalars are vectorized into this value. It is initialized to Null.
728  Value *VectorizedValue = nullptr;
729 
730  /// Do we need to gather this sequence ?
731  bool NeedToGather = false;
732 
733  /// Does this sequence require some shuffling?
734  SmallVector<unsigned, 4> ReuseShuffleIndices;
735 
736  /// Does this entry require reordering?
737  ArrayRef<unsigned> ReorderIndices;
738 
739  /// Points back to the VectorizableTree.
740  ///
741  /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
742  /// to be a pointer and needs to be able to initialize the child iterator.
743  /// Thus we need a reference back to the container to translate the indices
744  /// to entries.
745  std::vector<TreeEntry> &Container;
746 
747  /// The TreeEntry index containing the user of this entry. We can actually
748  /// have multiple users so the data structure is not truly a tree.
749  SmallVector<int, 1> UserTreeIndices;
750  };
751 
752  /// Create a new VectorizableTree entry.
753  void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
754  ArrayRef<unsigned> ReuseShuffleIndices = None,
755  ArrayRef<unsigned> ReorderIndices = None) {
756  VectorizableTree.emplace_back(VectorizableTree);
757  int idx = VectorizableTree.size() - 1;
758  TreeEntry *Last = &VectorizableTree[idx];
759  Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
760  Last->NeedToGather = !Vectorized;
761  Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
762  ReuseShuffleIndices.end());
763  Last->ReorderIndices = ReorderIndices;
764  if (Vectorized) {
765  for (int i = 0, e = VL.size(); i != e; ++i) {
766  assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
767  ScalarToTreeEntry[VL[i]] = idx;
768  }
769  } else {
770  MustGather.insert(VL.begin(), VL.end());
771  }
772 
773  if (UserTreeIdx >= 0)
774  Last->UserTreeIndices.push_back(UserTreeIdx);
775  UserTreeIdx = idx;
776  }
777 
778  /// -- Vectorization State --
779  /// Holds all of the tree entries.
780  std::vector<TreeEntry> VectorizableTree;
781 
782  TreeEntry *getTreeEntry(Value *V) {
783  auto I = ScalarToTreeEntry.find(V);
784  if (I != ScalarToTreeEntry.end())
785  return &VectorizableTree[I->second];
786  return nullptr;
787  }
788 
789  /// Maps a specific scalar to its tree entry.
790  SmallDenseMap<Value*, int> ScalarToTreeEntry;
791 
792  /// A list of scalars that we found that we need to keep as scalars.
793  ValueSet MustGather;
794 
795  /// This POD struct describes one external user in the vectorized tree.
796  struct ExternalUser {
797  ExternalUser(Value *S, llvm::User *U, int L)
798  : Scalar(S), User(U), Lane(L) {}
799 
800  // Which scalar in our function.
801  Value *Scalar;
802 
803  // Which user that uses the scalar.
804  llvm::User *User;
805 
806  // Which lane does the scalar belong to.
807  int Lane;
808  };
810 
811  /// Checks if two instructions may access the same memory.
812  ///
813  /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
814  /// is invariant in the calling loop.
815  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
816  Instruction *Inst2) {
817  // First check if the result is already in the cache.
818  AliasCacheKey key = std::make_pair(Inst1, Inst2);
819  Optional<bool> &result = AliasCache[key];
820  if (result.hasValue()) {
821  return result.getValue();
822  }
823  MemoryLocation Loc2 = getLocation(Inst2, AA);
824  bool aliased = true;
825  if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
826  // Do the alias check.
827  aliased = AA->alias(Loc1, Loc2);
828  }
829  // Store the result in the cache.
830  result = aliased;
831  return aliased;
832  }
833 
834  using AliasCacheKey = std::pair<Instruction *, Instruction *>;
835 
836  /// Cache for alias results.
837  /// TODO: consider moving this to the AliasAnalysis itself.
839 
840  /// Removes an instruction from its block and eventually deletes it.
841  /// It's like Instruction::eraseFromParent() except that the actual deletion
842  /// is delayed until BoUpSLP is destructed.
843  /// This is required to ensure that there are no incorrect collisions in the
844  /// AliasCache, which can happen if a new instruction is allocated at the
845  /// same address as a previously deleted instruction.
846  void eraseInstruction(Instruction *I) {
847  I->removeFromParent();
848  I->dropAllReferences();
849  DeletedInstructions.emplace_back(I);
850  }
851 
852  /// Temporary store for deleted instructions. Instructions will be deleted
853  /// eventually when the BoUpSLP is destructed.
854  SmallVector<unique_value, 8> DeletedInstructions;
855 
856  /// A list of values that need to extracted out of the tree.
857  /// This list holds pairs of (Internal Scalar : External User). External User
858  /// can be nullptr, it means that this Internal Scalar will be used later,
859  /// after vectorization.
860  UserList ExternalUses;
861 
862  /// Values used only by @llvm.assume calls.
864 
865  /// Holds all of the instructions that we gathered.
866  SetVector<Instruction *> GatherSeq;
867 
868  /// A list of blocks that we are going to CSE.
869  SetVector<BasicBlock *> CSEBlocks;
870 
871  /// Contains all scheduling relevant data for an instruction.
872  /// A ScheduleData either represents a single instruction or a member of an
873  /// instruction bundle (= a group of instructions which is combined into a
874  /// vector instruction).
875  struct ScheduleData {
876  // The initial value for the dependency counters. It means that the
877  // dependencies are not calculated yet.
878  enum { InvalidDeps = -1 };
879 
880  ScheduleData() = default;
881 
882  void init(int BlockSchedulingRegionID, Value *OpVal) {
883  FirstInBundle = this;
884  NextInBundle = nullptr;
885  NextLoadStore = nullptr;
886  IsScheduled = false;
887  SchedulingRegionID = BlockSchedulingRegionID;
888  UnscheduledDepsInBundle = UnscheduledDeps;
889  clearDependencies();
890  OpValue = OpVal;
891  }
892 
893  /// Returns true if the dependency information has been calculated.
894  bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
895 
896  /// Returns true for single instructions and for bundle representatives
897  /// (= the head of a bundle).
898  bool isSchedulingEntity() const { return FirstInBundle == this; }
899 
900  /// Returns true if it represents an instruction bundle and not only a
901  /// single instruction.
902  bool isPartOfBundle() const {
903  return NextInBundle != nullptr || FirstInBundle != this;
904  }
905 
906  /// Returns true if it is ready for scheduling, i.e. it has no more
907  /// unscheduled depending instructions/bundles.
908  bool isReady() const {
909  assert(isSchedulingEntity() &&
910  "can't consider non-scheduling entity for ready list");
911  return UnscheduledDepsInBundle == 0 && !IsScheduled;
912  }
913 
914  /// Modifies the number of unscheduled dependencies, also updating it for
915  /// the whole bundle.
916  int incrementUnscheduledDeps(int Incr) {
917  UnscheduledDeps += Incr;
918  return FirstInBundle->UnscheduledDepsInBundle += Incr;
919  }
920 
921  /// Sets the number of unscheduled dependencies to the number of
922  /// dependencies.
923  void resetUnscheduledDeps() {
924  incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
925  }
926 
927  /// Clears all dependency information.
928  void clearDependencies() {
929  Dependencies = InvalidDeps;
930  resetUnscheduledDeps();
931  MemoryDependencies.clear();
932  }
933 
934  void dump(raw_ostream &os) const {
935  if (!isSchedulingEntity()) {
936  os << "/ " << *Inst;
937  } else if (NextInBundle) {
938  os << '[' << *Inst;
939  ScheduleData *SD = NextInBundle;
940  while (SD) {
941  os << ';' << *SD->Inst;
942  SD = SD->NextInBundle;
943  }
944  os << ']';
945  } else {
946  os << *Inst;
947  }
948  }
949 
950  Instruction *Inst = nullptr;
951 
952  /// Points to the head in an instruction bundle (and always to this for
953  /// single instructions).
954  ScheduleData *FirstInBundle = nullptr;
955 
956  /// Single linked list of all instructions in a bundle. Null if it is a
957  /// single instruction.
958  ScheduleData *NextInBundle = nullptr;
959 
960  /// Single linked list of all memory instructions (e.g. load, store, call)
961  /// in the block - until the end of the scheduling region.
962  ScheduleData *NextLoadStore = nullptr;
963 
964  /// The dependent memory instructions.
965  /// This list is derived on demand in calculateDependencies().
966  SmallVector<ScheduleData *, 4> MemoryDependencies;
967 
968  /// This ScheduleData is in the current scheduling region if this matches
969  /// the current SchedulingRegionID of BlockScheduling.
970  int SchedulingRegionID = 0;
971 
972  /// Used for getting a "good" final ordering of instructions.
973  int SchedulingPriority = 0;
974 
975  /// The number of dependencies. Constitutes of the number of users of the
976  /// instruction plus the number of dependent memory instructions (if any).
977  /// This value is calculated on demand.
978  /// If InvalidDeps, the number of dependencies is not calculated yet.
979  int Dependencies = InvalidDeps;
980 
981  /// The number of dependencies minus the number of dependencies of scheduled
982  /// instructions. As soon as this is zero, the instruction/bundle gets ready
983  /// for scheduling.
984  /// Note that this is negative as long as Dependencies is not calculated.
985  int UnscheduledDeps = InvalidDeps;
986 
987  /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
988  /// single instructions.
989  int UnscheduledDepsInBundle = InvalidDeps;
990 
991  /// True if this instruction is scheduled (or considered as scheduled in the
992  /// dry-run).
993  bool IsScheduled = false;
994 
995  /// Opcode of the current instruction in the schedule data.
996  Value *OpValue = nullptr;
997  };
998 
999 #ifndef NDEBUG
1000  friend inline raw_ostream &operator<<(raw_ostream &os,
1001  const BoUpSLP::ScheduleData &SD) {
1002  SD.dump(os);
1003  return os;
1004  }
1005 #endif
1006 
1007  friend struct GraphTraits<BoUpSLP *>;
1008  friend struct DOTGraphTraits<BoUpSLP *>;
1009 
1010  /// Contains all scheduling data for a basic block.
1011  struct BlockScheduling {
1012  BlockScheduling(BasicBlock *BB)
1013  : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
1014 
1015  void clear() {
1016  ReadyInsts.clear();
1017  ScheduleStart = nullptr;
1018  ScheduleEnd = nullptr;
1019  FirstLoadStoreInRegion = nullptr;
1020  LastLoadStoreInRegion = nullptr;
1021 
1022  // Reduce the maximum schedule region size by the size of the
1023  // previous scheduling run.
1024  ScheduleRegionSizeLimit -= ScheduleRegionSize;
1025  if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1026  ScheduleRegionSizeLimit = MinScheduleRegionSize;
1027  ScheduleRegionSize = 0;
1028 
1029  // Make a new scheduling region, i.e. all existing ScheduleData is not
1030  // in the new region yet.
1031  ++SchedulingRegionID;
1032  }
1033 
1034  ScheduleData *getScheduleData(Value *V) {
1035  ScheduleData *SD = ScheduleDataMap[V];
1036  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1037  return SD;
1038  return nullptr;
1039  }
1040 
1041  ScheduleData *getScheduleData(Value *V, Value *Key) {
1042  if (V == Key)
1043  return getScheduleData(V);
1044  auto I = ExtraScheduleDataMap.find(V);
1045  if (I != ExtraScheduleDataMap.end()) {
1046  ScheduleData *SD = I->second[Key];
1047  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1048  return SD;
1049  }
1050  return nullptr;
1051  }
1052 
1053  bool isInSchedulingRegion(ScheduleData *SD) {
1054  return SD->SchedulingRegionID == SchedulingRegionID;
1055  }
1056 
1057  /// Marks an instruction as scheduled and puts all dependent ready
1058  /// instructions into the ready-list.
1059  template <typename ReadyListType>
1060  void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1061  SD->IsScheduled = true;
1062  DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1063 
1064  ScheduleData *BundleMember = SD;
1065  while (BundleMember) {
1066  if (BundleMember->Inst != BundleMember->OpValue) {
1067  BundleMember = BundleMember->NextInBundle;
1068  continue;
1069  }
1070  // Handle the def-use chain dependencies.
1071  for (Use &U : BundleMember->Inst->operands()) {
1072  auto *I = dyn_cast<Instruction>(U.get());
1073  if (!I)
1074  continue;
1075  doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1076  if (OpDef && OpDef->hasValidDependencies() &&
1077  OpDef->incrementUnscheduledDeps(-1) == 0) {
1078  // There are no more unscheduled dependencies after
1079  // decrementing, so we can put the dependent instruction
1080  // into the ready list.
1081  ScheduleData *DepBundle = OpDef->FirstInBundle;
1082  assert(!DepBundle->IsScheduled &&
1083  "already scheduled bundle gets ready");
1084  ReadyList.insert(DepBundle);
1085  DEBUG(dbgs()
1086  << "SLP: gets ready (def): " << *DepBundle << "\n");
1087  }
1088  });
1089  }
1090  // Handle the memory dependencies.
1091  for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1092  if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1093  // There are no more unscheduled dependencies after decrementing,
1094  // so we can put the dependent instruction into the ready list.
1095  ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1096  assert(!DepBundle->IsScheduled &&
1097  "already scheduled bundle gets ready");
1098  ReadyList.insert(DepBundle);
1099  DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle
1100  << "\n");
1101  }
1102  }
1103  BundleMember = BundleMember->NextInBundle;
1104  }
1105  }
1106 
1107  void doForAllOpcodes(Value *V,
1108  function_ref<void(ScheduleData *SD)> Action) {
1109  if (ScheduleData *SD = getScheduleData(V))
1110  Action(SD);
1111  auto I = ExtraScheduleDataMap.find(V);
1112  if (I != ExtraScheduleDataMap.end())
1113  for (auto &P : I->second)
1114  if (P.second->SchedulingRegionID == SchedulingRegionID)
1115  Action(P.second);
1116  }
1117 
1118  /// Put all instructions into the ReadyList which are ready for scheduling.
1119  template <typename ReadyListType>
1120  void initialFillReadyList(ReadyListType &ReadyList) {
1121  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1122  doForAllOpcodes(I, [&](ScheduleData *SD) {
1123  if (SD->isSchedulingEntity() && SD->isReady()) {
1124  ReadyList.insert(SD);
1125  DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n");
1126  }
1127  });
1128  }
1129  }
1130 
1131  /// Checks if a bundle of instructions can be scheduled, i.e. has no
1132  /// cyclic dependencies. This is only a dry-run, no instructions are
1133  /// actually moved at this stage.
1134  bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1135 
1136  /// Un-bundles a group of instructions.
1137  void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1138 
1139  /// Allocates schedule data chunk.
1140  ScheduleData *allocateScheduleDataChunks();
1141 
1142  /// Extends the scheduling region so that V is inside the region.
1143  /// \returns true if the region size is within the limit.
1144  bool extendSchedulingRegion(Value *V, Value *OpValue);
1145 
1146  /// Initialize the ScheduleData structures for new instructions in the
1147  /// scheduling region.
1148  void initScheduleData(Instruction *FromI, Instruction *ToI,
1149  ScheduleData *PrevLoadStore,
1150  ScheduleData *NextLoadStore);
1151 
1152  /// Updates the dependency information of a bundle and of all instructions/
1153  /// bundles which depend on the original bundle.
1154  void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1155  BoUpSLP *SLP);
1156 
1157  /// Sets all instruction in the scheduling region to un-scheduled.
1158  void resetSchedule();
1159 
1160  BasicBlock *BB;
1161 
1162  /// Simple memory allocation for ScheduleData.
1163  std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1164 
1165  /// The size of a ScheduleData array in ScheduleDataChunks.
1166  int ChunkSize;
1167 
1168  /// The allocator position in the current chunk, which is the last entry
1169  /// of ScheduleDataChunks.
1170  int ChunkPos;
1171 
1172  /// Attaches ScheduleData to Instruction.
1173  /// Note that the mapping survives during all vectorization iterations, i.e.
1174  /// ScheduleData structures are recycled.
1175  DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1176 
1177  /// Attaches ScheduleData to Instruction with the leading key.
1179  ExtraScheduleDataMap;
1180 
1181  struct ReadyList : SmallVector<ScheduleData *, 8> {
1182  void insert(ScheduleData *SD) { push_back(SD); }
1183  };
1184 
1185  /// The ready-list for scheduling (only used for the dry-run).
1186  ReadyList ReadyInsts;
1187 
1188  /// The first instruction of the scheduling region.
1189  Instruction *ScheduleStart = nullptr;
1190 
1191  /// The first instruction _after_ the scheduling region.
1192  Instruction *ScheduleEnd = nullptr;
1193 
1194  /// The first memory accessing instruction in the scheduling region
1195  /// (can be null).
1196  ScheduleData *FirstLoadStoreInRegion = nullptr;
1197 
1198  /// The last memory accessing instruction in the scheduling region
1199  /// (can be null).
1200  ScheduleData *LastLoadStoreInRegion = nullptr;
1201 
1202  /// The current size of the scheduling region.
1203  int ScheduleRegionSize = 0;
1204 
1205  /// The maximum size allowed for the scheduling region.
1206  int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1207 
1208  /// The ID of the scheduling region. For a new vectorization iteration this
1209  /// is incremented which "removes" all ScheduleData from the region.
1210  // Make sure that the initial SchedulingRegionID is greater than the
1211  // initial SchedulingRegionID in ScheduleData (which is 0).
1212  int SchedulingRegionID = 1;
1213  };
1214 
1215  /// Attaches the BlockScheduling structures to basic blocks.
1217 
1218  /// Performs the "real" scheduling. Done before vectorization is actually
1219  /// performed in a basic block.
1220  void scheduleBlock(BlockScheduling *BS);
1221 
1222  /// List of users to ignore during scheduling and that don't need extracting.
1223  ArrayRef<Value *> UserIgnoreList;
1224 
1226  /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
1227  /// sorted SmallVectors of unsigned.
1228  struct OrdersTypeDenseMapInfo {
1229  static OrdersType getEmptyKey() {
1230  OrdersType V;
1231  V.push_back(~1U);
1232  return V;
1233  }
1234 
1235  static OrdersType getTombstoneKey() {
1236  OrdersType V;
1237  V.push_back(~2U);
1238  return V;
1239  }
1240 
1241  static unsigned getHashValue(const OrdersType &V) {
1242  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1243  }
1244 
1245  static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
1246  return LHS == RHS;
1247  }
1248  };
1249 
1250  /// Contains orders of operations along with the number of bundles that have
1251  /// operations in this order. It stores only those orders that require
1252  /// reordering, if reordering is not required it is counted using \a
1253  /// NumOpsWantToKeepOriginalOrder.
1255  /// Number of bundles that do not require reordering.
1256  unsigned NumOpsWantToKeepOriginalOrder = 0;
1257 
1258  // Analysis and block reference.
1259  Function *F;
1260  ScalarEvolution *SE;
1261  TargetTransformInfo *TTI;
1262  TargetLibraryInfo *TLI;
1263  AliasAnalysis *AA;
1264  LoopInfo *LI;
1265  DominatorTree *DT;
1266  AssumptionCache *AC;
1267  DemandedBits *DB;
1268  const DataLayout *DL;
1270 
1271  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1272  unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1273 
1274  /// Instruction builder to construct the vectorized tree.
1275  IRBuilder<> Builder;
1276 
1277  /// A map of scalar integer values to the smallest bit width with which they
1278  /// can legally be represented. The values map to (width, signed) pairs,
1279  /// where "width" indicates the minimum bit width and "signed" is True if the
1280  /// value must be signed-extended, rather than zero-extended, back to its
1281  /// original width.
1283 };
1284 
1285 } // end namespace slpvectorizer
1286 
1287 template <> struct GraphTraits<BoUpSLP *> {
1288  using TreeEntry = BoUpSLP::TreeEntry;
1289 
1290  /// NodeRef has to be a pointer per the GraphWriter.
1291  using NodeRef = TreeEntry *;
1292 
1293  /// \brief Add the VectorizableTree to the index iterator to be able to return
1294  /// TreeEntry pointers.
1295  struct ChildIteratorType
1296  : public iterator_adaptor_base<ChildIteratorType,
1297  SmallVector<int, 1>::iterator> {
1298  std::vector<TreeEntry> &VectorizableTree;
1299 
1301  std::vector<TreeEntry> &VT)
1302  : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1303 
1304  NodeRef operator*() { return &VectorizableTree[*I]; }
1305  };
1306 
1307  static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1308 
1309  static ChildIteratorType child_begin(NodeRef N) {
1310  return {N->UserTreeIndices.begin(), N->Container};
1311  }
1312 
1313  static ChildIteratorType child_end(NodeRef N) {
1314  return {N->UserTreeIndices.end(), N->Container};
1315  }
1316 
1317  /// For the node iterator we just need to turn the TreeEntry iterator into a
1318  /// TreeEntry* iterator so that it dereferences to NodeRef.
1320 
1321  static nodes_iterator nodes_begin(BoUpSLP *R) {
1322  return nodes_iterator(R->VectorizableTree.begin());
1323  }
1324 
1325  static nodes_iterator nodes_end(BoUpSLP *R) {
1326  return nodes_iterator(R->VectorizableTree.end());
1327  }
1328 
1329  static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1330 };
1331 
1332 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1333  using TreeEntry = BoUpSLP::TreeEntry;
1334 
1336 
1337  std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1338  std::string Str;
1339  raw_string_ostream OS(Str);
1340  if (isSplat(Entry->Scalars)) {
1341  OS << "<splat> " << *Entry->Scalars[0];
1342  return Str;
1343  }
1344  for (auto V : Entry->Scalars) {
1345  OS << *V;
1346  if (std::any_of(
1347  R->ExternalUses.begin(), R->ExternalUses.end(),
1348  [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1349  OS << " <extract>";
1350  OS << "\n";
1351  }
1352  return Str;
1353  }
1354 
1355  static std::string getNodeAttributes(const TreeEntry *Entry,
1356  const BoUpSLP *) {
1357  if (Entry->NeedToGather)
1358  return "color=red";
1359  return "";
1360  }
1361 };
1362 
1363 } // end namespace llvm
1364 
1365 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1366  ArrayRef<Value *> UserIgnoreLst) {
1367  ExtraValueToDebugLocsMap ExternallyUsedValues;
1368  buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1369 }
1370 
1371 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1372  ExtraValueToDebugLocsMap &ExternallyUsedValues,
1373  ArrayRef<Value *> UserIgnoreLst) {
1374  deleteTree();
1375  UserIgnoreList = UserIgnoreLst;
1376  if (!allSameType(Roots))
1377  return;
1378  buildTree_rec(Roots, 0, -1);
1379 
1380  // Collect the values that we need to extract from the tree.
1381  for (TreeEntry &EIdx : VectorizableTree) {
1382  TreeEntry *Entry = &EIdx;
1383 
1384  // No need to handle users of gathered values.
1385  if (Entry->NeedToGather)
1386  continue;
1387 
1388  // For each lane:
1389  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1390  Value *Scalar = Entry->Scalars[Lane];
1391  int FoundLane = Lane;
1392  if (!Entry->ReuseShuffleIndices.empty()) {
1393  FoundLane =
1394  std::distance(Entry->ReuseShuffleIndices.begin(),
1395  llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1396  }
1397 
1398  // Check if the scalar is externally used as an extra arg.
1399  auto ExtI = ExternallyUsedValues.find(Scalar);
1400  if (ExtI != ExternallyUsedValues.end()) {
1401  DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<
1402  Lane << " from " << *Scalar << ".\n");
1403  ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1404  }
1405  for (User *U : Scalar->users()) {
1406  DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1407 
1408  Instruction *UserInst = dyn_cast<Instruction>(U);
1409  if (!UserInst)
1410  continue;
1411 
1412  // Skip in-tree scalars that become vectors
1413  if (TreeEntry *UseEntry = getTreeEntry(U)) {
1414  Value *UseScalar = UseEntry->Scalars[0];
1415  // Some in-tree scalars will remain as scalar in vectorized
1416  // instructions. If that is the case, the one in Lane 0 will
1417  // be used.
1418  if (UseScalar != U ||
1419  !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1420  DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1421  << ".\n");
1422  assert(!UseEntry->NeedToGather && "Bad state");
1423  continue;
1424  }
1425  }
1426 
1427  // Ignore users in the user ignore list.
1428  if (is_contained(UserIgnoreList, UserInst))
1429  continue;
1430 
1431  DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<
1432  Lane << " from " << *Scalar << ".\n");
1433  ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1434  }
1435  }
1436  }
1437 }
1438 
1439 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1440  int UserTreeIdx) {
1441  assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1442 
1443  InstructionsState S = getSameOpcode(VL);
1444  if (Depth == RecursionMaxDepth) {
1445  DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1446  newTreeEntry(VL, false, UserTreeIdx);
1447  return;
1448  }
1449 
1450  // Don't handle vectors.
1451  if (S.OpValue->getType()->isVectorTy()) {
1452  DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1453  newTreeEntry(VL, false, UserTreeIdx);
1454  return;
1455  }
1456 
1457  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1458  if (SI->getValueOperand()->getType()->isVectorTy()) {
1459  DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1460  newTreeEntry(VL, false, UserTreeIdx);
1461  return;
1462  }
1463 
1464  // If all of the operands are identical or constant we have a simple solution.
1465  if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1466  DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1467  newTreeEntry(VL, false, UserTreeIdx);
1468  return;
1469  }
1470 
1471  // We now know that this is a vector of instructions of the same type from
1472  // the same block.
1473 
1474  // Don't vectorize ephemeral values.
1475  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1476  if (EphValues.count(VL[i])) {
1477  DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1478  ") is ephemeral.\n");
1479  newTreeEntry(VL, false, UserTreeIdx);
1480  return;
1481  }
1482  }
1483 
1484  // Check if this is a duplicate of another entry.
1485  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1486  DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n");
1487  if (!E->isSame(VL)) {
1488  DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1489  newTreeEntry(VL, false, UserTreeIdx);
1490  return;
1491  }
1492  // Record the reuse of the tree node. FIXME, currently this is only used to
1493  // properly draw the graph rather than for the actual vectorization.
1494  E->UserTreeIndices.push_back(UserTreeIdx);
1495  DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n");
1496  return;
1497  }
1498 
1499  // Check that none of the instructions in the bundle are already in the tree.
1500  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1501  auto *I = dyn_cast<Instruction>(VL[i]);
1502  if (!I)
1503  continue;
1504  if (getTreeEntry(I)) {
1505  DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1506  ") is already in tree.\n");
1507  newTreeEntry(VL, false, UserTreeIdx);
1508  return;
1509  }
1510  }
1511 
1512  // If any of the scalars is marked as a value that needs to stay scalar, then
1513  // we need to gather the scalars.
1514  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1515  if (MustGather.count(VL[i])) {
1516  DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1517  newTreeEntry(VL, false, UserTreeIdx);
1518  return;
1519  }
1520  }
1521 
1522  // Check that all of the users of the scalars that we want to vectorize are
1523  // schedulable.
1524  auto *VL0 = cast<Instruction>(S.OpValue);
1525  BasicBlock *BB = VL0->getParent();
1526 
1527  if (!DT->isReachableFromEntry(BB)) {
1528  // Don't go into unreachable blocks. They may contain instructions with
1529  // dependency cycles which confuse the final scheduling.
1530  DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1531  newTreeEntry(VL, false, UserTreeIdx);
1532  return;
1533  }
1534 
1535  // Check that every instruction appears once in this bundle.
1536  SmallVector<unsigned, 4> ReuseShuffleIndicies;
1537  SmallVector<Value *, 4> UniqueValues;
1538  DenseMap<Value *, unsigned> UniquePositions;
1539  for (Value *V : VL) {
1540  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1541  ReuseShuffleIndicies.emplace_back(Res.first->second);
1542  if (Res.second)
1543  UniqueValues.emplace_back(V);
1544  }
1545  if (UniqueValues.size() == VL.size()) {
1546  ReuseShuffleIndicies.clear();
1547  } else {
1548  DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
1549  if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1550  DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1551  newTreeEntry(VL, false, UserTreeIdx);
1552  return;
1553  }
1554  VL = UniqueValues;
1555  }
1556 
1557  auto &BSRef = BlocksSchedules[BB];
1558  if (!BSRef)
1559  BSRef = llvm::make_unique<BlockScheduling>(BB);
1560 
1561  BlockScheduling &BS = *BSRef.get();
1562 
1563  if (!BS.tryScheduleBundle(VL, this, VL0)) {
1564  DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1565  assert((!BS.getScheduleData(VL0) ||
1566  !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1567  "tryScheduleBundle should cancelScheduling on failure");
1568  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1569  return;
1570  }
1571  DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1572 
1573  unsigned ShuffleOrOp = S.IsAltShuffle ?
1574  (unsigned) Instruction::ShuffleVector : S.Opcode;
1575  switch (ShuffleOrOp) {
1576  case Instruction::PHI: {
1577  PHINode *PH = dyn_cast<PHINode>(VL0);
1578 
1579  // Check for terminator values (e.g. invoke).
1580  for (unsigned j = 0; j < VL.size(); ++j)
1581  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1583  cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1584  if (Term) {
1585  DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n");
1586  BS.cancelScheduling(VL, VL0);
1587  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1588  return;
1589  }
1590  }
1591 
1592  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1593  DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1594 
1595  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1596  ValueList Operands;
1597  // Prepare the operand vector.
1598  for (Value *j : VL)
1599  Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1600  PH->getIncomingBlock(i)));
1601 
1602  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1603  }
1604  return;
1605  }
1606  case Instruction::ExtractValue:
1607  case Instruction::ExtractElement: {
1608  OrdersType CurrentOrder;
1609  bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
1610  if (Reuse) {
1611  DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
1612  ++NumOpsWantToKeepOriginalOrder;
1613  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1614  ReuseShuffleIndicies);
1615  return;
1616  }
1617  if (!CurrentOrder.empty()) {
1618  DEBUG({
1619  dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
1620  "with order";
1621  for (unsigned Idx : CurrentOrder)
1622  dbgs() << " " << Idx;
1623  dbgs() << "\n";
1624  });
1625  // Insert new order with initial value 0, if it does not exist,
1626  // otherwise return the iterator to the existing one.
1627  auto StoredCurrentOrderAndNum =
1628  NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1629  ++StoredCurrentOrderAndNum->getSecond();
1630  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx, ReuseShuffleIndicies,
1631  StoredCurrentOrderAndNum->getFirst());
1632  return;
1633  }
1634  DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
1635  newTreeEntry(VL, /*Vectorized=*/false, UserTreeIdx, ReuseShuffleIndicies);
1636  BS.cancelScheduling(VL, VL0);
1637  return;
1638  }
1639  case Instruction::Load: {
1640  // Check that a vectorized load would load the same memory as a scalar
1641  // load. For example, we don't want to vectorize loads that are smaller
1642  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1643  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1644  // from such a struct, we read/write packed bits disagreeing with the
1645  // unvectorized version.
1646  Type *ScalarTy = VL0->getType();
1647 
1648  if (DL->getTypeSizeInBits(ScalarTy) !=
1649  DL->getTypeAllocSizeInBits(ScalarTy)) {
1650  BS.cancelScheduling(VL, VL0);
1651  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1652  DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1653  return;
1654  }
1655 
1656  // Make sure all loads in the bundle are simple - we can't vectorize
1657  // atomic or volatile loads.
1658  SmallVector<Value *, 4> PointerOps(VL.size());
1659  auto POIter = PointerOps.begin();
1660  for (Value *V : VL) {
1661  auto *L = cast<LoadInst>(V);
1662  if (!L->isSimple()) {
1663  BS.cancelScheduling(VL, VL0);
1664  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1665  DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1666  return;
1667  }
1668  *POIter = L->getPointerOperand();
1669  ++POIter;
1670  }
1671 
1672  OrdersType CurrentOrder;
1673  // Check the order of pointer operands.
1674  if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
1675  Value *Ptr0;
1676  Value *PtrN;
1677  if (CurrentOrder.empty()) {
1678  Ptr0 = PointerOps.front();
1679  PtrN = PointerOps.back();
1680  } else {
1681  Ptr0 = PointerOps[CurrentOrder.front()];
1682  PtrN = PointerOps[CurrentOrder.back()];
1683  }
1684  const SCEV *Scev0 = SE->getSCEV(Ptr0);
1685  const SCEV *ScevN = SE->getSCEV(PtrN);
1686  const auto *Diff =
1687  dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
1688  uint64_t Size = DL->getTypeAllocSize(ScalarTy);
1689  // Check that the sorted loads are consecutive.
1690  if (Diff && Diff->getAPInt().getZExtValue() == (VL.size() - 1) * Size) {
1691  if (CurrentOrder.empty()) {
1692  // Original loads are consecutive and does not require reordering.
1693  ++NumOpsWantToKeepOriginalOrder;
1694  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1695  ReuseShuffleIndicies);
1696  DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1697  } else {
1698  // Need to reorder.
1699  auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1700  ++I->getSecond();
1701  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1702  ReuseShuffleIndicies, I->getFirst());
1703  DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
1704  }
1705  return;
1706  }
1707  }
1708 
1709  DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1710  BS.cancelScheduling(VL, VL0);
1711  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1712  return;
1713  }
1714  case Instruction::ZExt:
1715  case Instruction::SExt:
1716  case Instruction::FPToUI:
1717  case Instruction::FPToSI:
1718  case Instruction::FPExt:
1719  case Instruction::PtrToInt:
1720  case Instruction::IntToPtr:
1721  case Instruction::SIToFP:
1722  case Instruction::UIToFP:
1723  case Instruction::Trunc:
1724  case Instruction::FPTrunc:
1725  case Instruction::BitCast: {
1726  Type *SrcTy = VL0->getOperand(0)->getType();
1727  for (unsigned i = 0; i < VL.size(); ++i) {
1728  Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1729  if (Ty != SrcTy || !isValidElementType(Ty)) {
1730  BS.cancelScheduling(VL, VL0);
1731  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1732  DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n");
1733  return;
1734  }
1735  }
1736  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1737  DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1738 
1739  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1740  ValueList Operands;
1741  // Prepare the operand vector.
1742  for (Value *j : VL)
1743  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1744 
1745  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1746  }
1747  return;
1748  }
1749  case Instruction::ICmp:
1750  case Instruction::FCmp: {
1751  // Check that all of the compares have the same predicate.
1752  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1753  Type *ComparedTy = VL0->getOperand(0)->getType();
1754  for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1755  CmpInst *Cmp = cast<CmpInst>(VL[i]);
1756  if (Cmp->getPredicate() != P0 ||
1757  Cmp->getOperand(0)->getType() != ComparedTy) {
1758  BS.cancelScheduling(VL, VL0);
1759  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1760  DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
1761  return;
1762  }
1763  }
1764 
1765  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1766  DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1767 
1768  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1769  ValueList Operands;
1770  // Prepare the operand vector.
1771  for (Value *j : VL)
1772  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1773 
1774  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1775  }
1776  return;
1777  }
1778  case Instruction::Select:
1779  case Instruction::Add:
1780  case Instruction::FAdd:
1781  case Instruction::Sub:
1782  case Instruction::FSub:
1783  case Instruction::Mul:
1784  case Instruction::FMul:
1785  case Instruction::UDiv:
1786  case Instruction::SDiv:
1787  case Instruction::FDiv:
1788  case Instruction::URem:
1789  case Instruction::SRem:
1790  case Instruction::FRem:
1791  case Instruction::Shl:
1792  case Instruction::LShr:
1793  case Instruction::AShr:
1794  case Instruction::And:
1795  case Instruction::Or:
1796  case Instruction::Xor:
1797  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1798  DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1799 
1800  // Sort operands of the instructions so that each side is more likely to
1801  // have the same opcode.
1802  if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1803  ValueList Left, Right;
1804  reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1805  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1806  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1807  return;
1808  }
1809 
1810  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1811  ValueList Operands;
1812  // Prepare the operand vector.
1813  for (Value *j : VL)
1814  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1815 
1816  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1817  }
1818  return;
1819 
1820  case Instruction::GetElementPtr: {
1821  // We don't combine GEPs with complicated (nested) indexing.
1822  for (unsigned j = 0; j < VL.size(); ++j) {
1823  if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1824  DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1825  BS.cancelScheduling(VL, VL0);
1826  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1827  return;
1828  }
1829  }
1830 
1831  // We can't combine several GEPs into one vector if they operate on
1832  // different types.
1833  Type *Ty0 = VL0->getOperand(0)->getType();
1834  for (unsigned j = 0; j < VL.size(); ++j) {
1835  Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1836  if (Ty0 != CurTy) {
1837  DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
1838  BS.cancelScheduling(VL, VL0);
1839  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1840  return;
1841  }
1842  }
1843 
1844  // We don't combine GEPs with non-constant indexes.
1845  for (unsigned j = 0; j < VL.size(); ++j) {
1846  auto Op = cast<Instruction>(VL[j])->getOperand(1);
1847  if (!isa<ConstantInt>(Op)) {
1848  DEBUG(
1849  dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1850  BS.cancelScheduling(VL, VL0);
1851  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1852  return;
1853  }
1854  }
1855 
1856  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1857  DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1858  for (unsigned i = 0, e = 2; i < e; ++i) {
1859  ValueList Operands;
1860  // Prepare the operand vector.
1861  for (Value *j : VL)
1862  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1863 
1864  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1865  }
1866  return;
1867  }
1868  case Instruction::Store: {
1869  // Check if the stores are consecutive or of we need to swizzle them.
1870  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1871  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1872  BS.cancelScheduling(VL, VL0);
1873  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1874  DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1875  return;
1876  }
1877 
1878  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1879  DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1880 
1881  ValueList Operands;
1882  for (Value *j : VL)
1883  Operands.push_back(cast<Instruction>(j)->getOperand(0));
1884 
1885  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1886  return;
1887  }
1888  case Instruction::Call: {
1889  // Check if the calls are all to the same vectorizable intrinsic.
1890  CallInst *CI = cast<CallInst>(VL0);
1891  // Check if this is an Intrinsic call or something that can be
1892  // represented by an intrinsic call
1894  if (!isTriviallyVectorizable(ID)) {
1895  BS.cancelScheduling(VL, VL0);
1896  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1897  DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1898  return;
1899  }
1900  Function *Int = CI->getCalledFunction();
1901  Value *A1I = nullptr;
1902  if (hasVectorInstrinsicScalarOpd(ID, 1))
1903  A1I = CI->getArgOperand(1);
1904  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1905  CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1906  if (!CI2 || CI2->getCalledFunction() != Int ||
1907  getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1908  !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1909  BS.cancelScheduling(VL, VL0);
1910  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1911  DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1912  << "\n");
1913  return;
1914  }
1915  // ctlz,cttz and powi are special intrinsics whose second argument
1916  // should be same in order for them to be vectorized.
1917  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1918  Value *A1J = CI2->getArgOperand(1);
1919  if (A1I != A1J) {
1920  BS.cancelScheduling(VL, VL0);
1921  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1922  DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1923  << " argument "<< A1I<<"!=" << A1J
1924  << "\n");
1925  return;
1926  }
1927  }
1928  // Verify that the bundle operands are identical between the two calls.
1929  if (CI->hasOperandBundles() &&
1931  CI->op_begin() + CI->getBundleOperandsEndIndex(),
1932  CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1933  BS.cancelScheduling(VL, VL0);
1934  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1935  DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="
1936  << *VL[i] << '\n');
1937  return;
1938  }
1939  }
1940 
1941  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1942  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1943  ValueList Operands;
1944  // Prepare the operand vector.
1945  for (Value *j : VL) {
1946  CallInst *CI2 = dyn_cast<CallInst>(j);
1947  Operands.push_back(CI2->getArgOperand(i));
1948  }
1949  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1950  }
1951  return;
1952  }
1953  case Instruction::ShuffleVector:
1954  // If this is not an alternate sequence of opcode like add-sub
1955  // then do not vectorize this instruction.
1956  if (!S.IsAltShuffle) {
1957  BS.cancelScheduling(VL, VL0);
1958  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1959  DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1960  return;
1961  }
1962  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1963  DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1964 
1965  // Reorder operands if reordering would enable vectorization.
1966  if (isa<BinaryOperator>(VL0)) {
1967  ValueList Left, Right;
1968  reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1969  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1970  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1971  return;
1972  }
1973 
1974  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1975  ValueList Operands;
1976  // Prepare the operand vector.
1977  for (Value *j : VL)
1978  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1979 
1980  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1981  }
1982  return;
1983 
1984  default:
1985  BS.cancelScheduling(VL, VL0);
1986  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1987  DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1988  return;
1989  }
1990 }
1991 
1992 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1993  unsigned N;
1994  Type *EltTy;
1995  auto *ST = dyn_cast<StructType>(T);
1996  if (ST) {
1997  N = ST->getNumElements();
1998  EltTy = *ST->element_begin();
1999  } else {
2000  N = cast<ArrayType>(T)->getNumElements();
2001  EltTy = cast<ArrayType>(T)->getElementType();
2002  }
2003  if (!isValidElementType(EltTy))
2004  return 0;
2005  uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
2006  if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
2007  return 0;
2008  if (ST) {
2009  // Check that struct is homogeneous.
2010  for (const auto *Ty : ST->elements())
2011  if (Ty != EltTy)
2012  return 0;
2013  }
2014  return N;
2015 }
2016 
2017 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
2018  SmallVectorImpl<unsigned> &CurrentOrder) const {
2019  Instruction *E0 = cast<Instruction>(OpValue);
2020  assert(E0->getOpcode() == Instruction::ExtractElement ||
2021  E0->getOpcode() == Instruction::ExtractValue);
2022  assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode");
2023  // Check if all of the extracts come from the same vector and from the
2024  // correct offset.
2025  Value *Vec = E0->getOperand(0);
2026 
2027  CurrentOrder.clear();
2028 
2029  // We have to extract from a vector/aggregate with the same number of elements.
2030  unsigned NElts;
2031  if (E0->getOpcode() == Instruction::ExtractValue) {
2032  const DataLayout &DL = E0->getModule()->getDataLayout();
2033  NElts = canMapToVector(Vec->getType(), DL);
2034  if (!NElts)
2035  return false;
2036  // Check if load can be rewritten as load of vector.
2037  LoadInst *LI = dyn_cast<LoadInst>(Vec);
2038  if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2039  return false;
2040  } else {
2041  NElts = Vec->getType()->getVectorNumElements();
2042  }
2043 
2044  if (NElts != VL.size())
2045  return false;
2046 
2047  // Check that all of the indices extract from the correct offset.
2048  bool ShouldKeepOrder = true;
2049  unsigned E = VL.size();
2050  // Assign to all items the initial value E + 1 so we can check if the extract
2051  // instruction index was used already.
2052  // Also, later we can check that all the indices are used and we have a
2053  // consecutive access in the extract instructions, by checking that no
2054  // element of CurrentOrder still has value E + 1.
2055  CurrentOrder.assign(E, E + 1);
2056  unsigned I = 0;
2057  for (; I < E; ++I) {
2058  auto *Inst = cast<Instruction>(VL[I]);
2059  if (Inst->getOperand(0) != Vec)
2060  break;
2061  Optional<unsigned> Idx = getExtractIndex(Inst);
2062  if (!Idx)
2063  break;
2064  const unsigned ExtIdx = *Idx;
2065  if (ExtIdx != I) {
2066  if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
2067  break;
2068  ShouldKeepOrder = false;
2069  CurrentOrder[ExtIdx] = I;
2070  } else {
2071  if (CurrentOrder[I] != E + 1)
2072  break;
2073  CurrentOrder[I] = I;
2074  }
2075  }
2076  if (I < E) {
2077  CurrentOrder.clear();
2078  return false;
2079  }
2080 
2081  return ShouldKeepOrder;
2082 }
2083 
2084 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2085  return I->hasOneUse() ||
2086  std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2087  return ScalarToTreeEntry.count(U) > 0;
2088  });
2089 }
2090 
2091 int BoUpSLP::getEntryCost(TreeEntry *E) {
2092  ArrayRef<Value*> VL = E->Scalars;
2093 
2094  Type *ScalarTy = VL[0]->getType();
2095  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2096  ScalarTy = SI->getValueOperand()->getType();
2097  else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2098  ScalarTy = CI->getOperand(0)->getType();
2099  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2100 
2101  // If we have computed a smaller type for the expression, update VecTy so
2102  // that the costs will be accurate.
2103  if (MinBWs.count(VL[0]))
2104  VecTy = VectorType::get(
2105  IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2106 
2107  unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2108  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2109  int ReuseShuffleCost = 0;
2110  if (NeedToShuffleReuses) {
2111  ReuseShuffleCost =
2112  TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2113  }
2114  if (E->NeedToGather) {
2115  if (allConstant(VL))
2116  return 0;
2117  if (isSplat(VL)) {
2118  return ReuseShuffleCost +
2119  TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2120  }
2121  if (getSameOpcode(VL).Opcode == Instruction::ExtractElement &&
2122  allSameType(VL) && allSameBlock(VL)) {
2124  if (ShuffleKind.hasValue()) {
2125  int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2126  for (auto *V : VL) {
2127  // If all users of instruction are going to be vectorized and this
2128  // instruction itself is not going to be vectorized, consider this
2129  // instruction as dead and remove its cost from the final cost of the
2130  // vectorized tree.
2131  if (areAllUsersVectorized(cast<Instruction>(V)) &&
2132  !ScalarToTreeEntry.count(V)) {
2133  auto *IO = cast<ConstantInt>(
2134  cast<ExtractElementInst>(V)->getIndexOperand());
2135  Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2136  IO->getZExtValue());
2137  }
2138  }
2139  return ReuseShuffleCost + Cost;
2140  }
2141  }
2142  return ReuseShuffleCost + getGatherCost(VL);
2143  }
2144  InstructionsState S = getSameOpcode(VL);
2145  assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2146  Instruction *VL0 = cast<Instruction>(S.OpValue);
2147  unsigned ShuffleOrOp = S.IsAltShuffle ?
2148  (unsigned) Instruction::ShuffleVector : S.Opcode;
2149  switch (ShuffleOrOp) {
2150  case Instruction::PHI:
2151  return 0;
2152 
2153  case Instruction::ExtractValue:
2154  case Instruction::ExtractElement:
2155  if (NeedToShuffleReuses) {
2156  unsigned Idx = 0;
2157  for (unsigned I : E->ReuseShuffleIndices) {
2158  if (ShuffleOrOp == Instruction::ExtractElement) {
2159  auto *IO = cast<ConstantInt>(
2160  cast<ExtractElementInst>(VL[I])->getIndexOperand());
2161  Idx = IO->getZExtValue();
2162  ReuseShuffleCost -= TTI->getVectorInstrCost(
2163  Instruction::ExtractElement, VecTy, Idx);
2164  } else {
2165  ReuseShuffleCost -= TTI->getVectorInstrCost(
2166  Instruction::ExtractElement, VecTy, Idx);
2167  ++Idx;
2168  }
2169  }
2170  Idx = ReuseShuffleNumbers;
2171  for (Value *V : VL) {
2172  if (ShuffleOrOp == Instruction::ExtractElement) {
2173  auto *IO = cast<ConstantInt>(
2174  cast<ExtractElementInst>(V)->getIndexOperand());
2175  Idx = IO->getZExtValue();
2176  } else {
2177  --Idx;
2178  }
2179  ReuseShuffleCost +=
2180  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2181  }
2182  }
2183  if (!E->NeedToGather) {
2184  int DeadCost = ReuseShuffleCost;
2185  if (!E->ReorderIndices.empty()) {
2186  // TODO: Merge this shuffle with the ReuseShuffleCost.
2187  DeadCost += TTI->getShuffleCost(
2189  }
2190  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2191  Instruction *E = cast<Instruction>(VL[i]);
2192  // If all users are going to be vectorized, instruction can be
2193  // considered as dead.
2194  // The same, if have only one user, it will be vectorized for sure.
2195  if (areAllUsersVectorized(E)) {
2196  // Take credit for instruction that will become dead.
2197  if (E->hasOneUse()) {
2198  Instruction *Ext = E->user_back();
2199  if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2200  all_of(Ext->users(),
2201  [](User *U) { return isa<GetElementPtrInst>(U); })) {
2202  // Use getExtractWithExtendCost() to calculate the cost of
2203  // extractelement/ext pair.
2204  DeadCost -= TTI->getExtractWithExtendCost(
2205  Ext->getOpcode(), Ext->getType(), VecTy, i);
2206  // Add back the cost of s|zext which is subtracted seperately.
2207  DeadCost += TTI->getCastInstrCost(
2208  Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
2209  continue;
2210  }
2211  }
2212  DeadCost -=
2213  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2214  }
2215  }
2216  return DeadCost;
2217  }
2218  return ReuseShuffleCost + getGatherCost(VL);
2219 
2220  case Instruction::ZExt:
2221  case Instruction::SExt:
2222  case Instruction::FPToUI:
2223  case Instruction::FPToSI:
2224  case Instruction::FPExt:
2225  case Instruction::PtrToInt:
2226  case Instruction::IntToPtr:
2227  case Instruction::SIToFP:
2228  case Instruction::UIToFP:
2229  case Instruction::Trunc:
2230  case Instruction::FPTrunc:
2231  case Instruction::BitCast: {
2232  Type *SrcTy = VL0->getOperand(0)->getType();
2233  if (NeedToShuffleReuses) {
2234  ReuseShuffleCost -=
2235  (ReuseShuffleNumbers - VL.size()) *
2236  TTI->getCastInstrCost(S.Opcode, ScalarTy, SrcTy, VL0);
2237  }
2238 
2239  // Calculate the cost of this instruction.
2240  int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2241  VL0->getType(), SrcTy, VL0);
2242 
2243  VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2244  int VecCost = 0;
2245  // Check if the values are candidates to demote.
2246  if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2247  VecCost = ReuseShuffleCost +
2248  TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2249  }
2250  return VecCost - ScalarCost;
2251  }
2252  case Instruction::FCmp:
2253  case Instruction::ICmp:
2254  case Instruction::Select: {
2255  // Calculate the cost of this instruction.
2256  if (NeedToShuffleReuses) {
2257  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2258  TTI->getCmpSelInstrCost(S.Opcode, ScalarTy,
2259  Builder.getInt1Ty(), VL0);
2260  }
2261  VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2262  int ScalarCost = VecTy->getNumElements() *
2263  TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2264  int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2265  return ReuseShuffleCost + VecCost - ScalarCost;
2266  }
2267  case Instruction::Add:
2268  case Instruction::FAdd:
2269  case Instruction::Sub:
2270  case Instruction::FSub:
2271  case Instruction::Mul:
2272  case Instruction::FMul:
2273  case Instruction::UDiv:
2274  case Instruction::SDiv:
2275  case Instruction::FDiv:
2276  case Instruction::URem:
2277  case Instruction::SRem:
2278  case Instruction::FRem:
2279  case Instruction::Shl:
2280  case Instruction::LShr:
2281  case Instruction::AShr:
2282  case Instruction::And:
2283  case Instruction::Or:
2284  case Instruction::Xor: {
2285  // Certain instructions can be cheaper to vectorize if they have a
2286  // constant second vector operand.
2295 
2296  // If all operands are exactly the same ConstantInt then set the
2297  // operand kind to OK_UniformConstantValue.
2298  // If instead not all operands are constants, then set the operand kind
2299  // to OK_AnyValue. If all operands are constants but not the same,
2300  // then set the operand kind to OK_NonUniformConstantValue.
2301  ConstantInt *CInt = nullptr;
2302  for (unsigned i = 0; i < VL.size(); ++i) {
2303  const Instruction *I = cast<Instruction>(VL[i]);
2304  if (!isa<ConstantInt>(I->getOperand(1))) {
2306  break;
2307  }
2308  if (i == 0) {
2309  CInt = cast<ConstantInt>(I->getOperand(1));
2310  continue;
2311  }
2313  CInt != cast<ConstantInt>(I->getOperand(1)))
2315  }
2316  // FIXME: Currently cost of model modification for division by power of
2317  // 2 is handled for X86 and AArch64. Add support for other targets.
2318  if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2319  CInt->getValue().isPowerOf2())
2321 
2323  if (NeedToShuffleReuses) {
2324  ReuseShuffleCost -=
2325  (ReuseShuffleNumbers - VL.size()) *
2326  TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2327  Op2VP, Operands);
2328  }
2329  int ScalarCost =
2330  VecTy->getNumElements() *
2331  TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2332  Op2VP, Operands);
2333  int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2334  Op1VP, Op2VP, Operands);
2335  return ReuseShuffleCost + VecCost - ScalarCost;
2336  }
2337  case Instruction::GetElementPtr: {
2342 
2343  if (NeedToShuffleReuses) {
2344  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2345  TTI->getArithmeticInstrCost(Instruction::Add,
2346  ScalarTy, Op1VK, Op2VK);
2347  }
2348  int ScalarCost =
2349  VecTy->getNumElements() *
2350  TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2351  int VecCost =
2352  TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2353 
2354  return ReuseShuffleCost + VecCost - ScalarCost;
2355  }
2356  case Instruction::Load: {
2357  // Cost of wide load - cost of scalar loads.
2358  unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2359  if (NeedToShuffleReuses) {
2360  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2361  TTI->getMemoryOpCost(Instruction::Load, ScalarTy,
2362  alignment, 0, VL0);
2363  }
2364  int ScalarLdCost = VecTy->getNumElements() *
2365  TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2366  int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2367  VecTy, alignment, 0, VL0);
2368  if (!E->ReorderIndices.empty()) {
2369  // TODO: Merge this shuffle with the ReuseShuffleCost.
2370  VecLdCost += TTI->getShuffleCost(
2372  }
2373  return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2374  }
2375  case Instruction::Store: {
2376  // We know that we can merge the stores. Calculate the cost.
2377  unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2378  if (NeedToShuffleReuses) {
2379  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2380  TTI->getMemoryOpCost(Instruction::Store, ScalarTy,
2381  alignment, 0, VL0);
2382  }
2383  int ScalarStCost = VecTy->getNumElements() *
2384  TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2385  int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2386  VecTy, alignment, 0, VL0);
2387  return ReuseShuffleCost + VecStCost - ScalarStCost;
2388  }
2389  case Instruction::Call: {
2390  CallInst *CI = cast<CallInst>(VL0);
2392 
2393  // Calculate the cost of the scalar and vector calls.
2394  SmallVector<Type*, 4> ScalarTys;
2395  for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2396  ScalarTys.push_back(CI->getArgOperand(op)->getType());
2397 
2398  FastMathFlags FMF;
2399  if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2400  FMF = FPMO->getFastMathFlags();
2401 
2402  if (NeedToShuffleReuses) {
2403  ReuseShuffleCost -=
2404  (ReuseShuffleNumbers - VL.size()) *
2405  TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2406  }
2407  int ScalarCallCost = VecTy->getNumElements() *
2408  TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2409 
2411  int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2412  VecTy->getNumElements());
2413 
2414  DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCost
2415  << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2416  << " for " << *CI << "\n");
2417 
2418  return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2419  }
2420  case Instruction::ShuffleVector: {
2425  int ScalarCost = 0;
2426  if (NeedToShuffleReuses) {
2427  for (unsigned Idx : E->ReuseShuffleIndices) {
2428  Instruction *I = cast<Instruction>(VL[Idx]);
2429  if (!I)
2430  continue;
2431  ReuseShuffleCost -= TTI->getArithmeticInstrCost(
2432  I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2433  }
2434  for (Value *V : VL) {
2435  Instruction *I = cast<Instruction>(V);
2436  if (!I)
2437  continue;
2438  ReuseShuffleCost += TTI->getArithmeticInstrCost(
2439  I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2440  }
2441  }
2442  int VecCost = 0;
2443  for (Value *i : VL) {
2444  Instruction *I = cast<Instruction>(i);
2445  if (!I)
2446  break;
2447  ScalarCost +=
2448  TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2449  }
2450  // VecCost is equal to sum of the cost of creating 2 vectors
2451  // and the cost of creating shuffle.
2452  Instruction *I0 = cast<Instruction>(VL[0]);
2453  VecCost =
2454  TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2455  Instruction *I1 = cast<Instruction>(VL[1]);
2456  VecCost +=
2457  TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2458  VecCost +=
2459  TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2460  return ReuseShuffleCost + VecCost - ScalarCost;
2461  }
2462  default:
2463  llvm_unreachable("Unknown instruction");
2464  }
2465 }
2466 
2467 bool BoUpSLP::isFullyVectorizableTinyTree() {
2468  DEBUG(dbgs() << "SLP: Check whether the tree with height " <<
2469  VectorizableTree.size() << " is fully vectorizable .\n");
2470 
2471  // We only handle trees of heights 1 and 2.
2472  if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2473  return true;
2474 
2475  if (VectorizableTree.size() != 2)
2476  return false;
2477 
2478  // Handle splat and all-constants stores.
2479  if (!VectorizableTree[0].NeedToGather &&
2480  (allConstant(VectorizableTree[1].Scalars) ||
2481  isSplat(VectorizableTree[1].Scalars)))
2482  return true;
2483 
2484  // Gathering cost would be too much for tiny trees.
2485  if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2486  return false;
2487 
2488  return true;
2489 }
2490 
2491 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2492  // We can vectorize the tree if its size is greater than or equal to the
2493  // minimum size specified by the MinTreeSize command line option.
2494  if (VectorizableTree.size() >= MinTreeSize)
2495  return false;
2496 
2497  // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2498  // can vectorize it if we can prove it fully vectorizable.
2499  if (isFullyVectorizableTinyTree())
2500  return false;
2501 
2502  assert(VectorizableTree.empty()
2503  ? ExternalUses.empty()
2504  : true && "We shouldn't have any external users");
2505 
2506  // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2507  // vectorizable.
2508  return true;
2509 }
2510 
2511 int BoUpSLP::getSpillCost() {
2512  // Walk from the bottom of the tree to the top, tracking which values are
2513  // live. When we see a call instruction that is not part of our tree,
2514  // query TTI to see if there is a cost to keeping values live over it
2515  // (for example, if spills and fills are required).
2516  unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2517  int Cost = 0;
2518 
2519  SmallPtrSet<Instruction*, 4> LiveValues;
2520  Instruction *PrevInst = nullptr;
2521 
2522  for (const auto &N : VectorizableTree) {
2523  Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2524  if (!Inst)
2525  continue;
2526 
2527  if (!PrevInst) {
2528  PrevInst = Inst;
2529  continue;
2530  }
2531 
2532  // Update LiveValues.
2533  LiveValues.erase(PrevInst);
2534  for (auto &J : PrevInst->operands()) {
2535  if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2536  LiveValues.insert(cast<Instruction>(&*J));
2537  }
2538 
2539  DEBUG({
2540  dbgs() << "SLP: #LV: " << LiveValues.size();
2541  for (auto *X : LiveValues)
2542  dbgs() << " " << X->getName();
2543  dbgs() << ", Looking at ";
2544  Inst->dump();
2545  });
2546 
2547  // Now find the sequence of instructions between PrevInst and Inst.
2549  PrevInstIt =
2550  PrevInst->getIterator().getReverse();
2551  while (InstIt != PrevInstIt) {
2552  if (PrevInstIt == PrevInst->getParent()->rend()) {
2553  PrevInstIt = Inst->getParent()->rbegin();
2554  continue;
2555  }
2556 
2557  if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2559  for (auto *II : LiveValues)
2560  V.push_back(VectorType::get(II->getType(), BundleWidth));
2561  Cost += TTI->getCostOfKeepingLiveOverCall(V);
2562  }
2563 
2564  ++PrevInstIt;
2565  }
2566 
2567  PrevInst = Inst;
2568  }
2569 
2570  return Cost;
2571 }
2572 
2573 int BoUpSLP::getTreeCost() {
2574  int Cost = 0;
2575  DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<
2576  VectorizableTree.size() << ".\n");
2577 
2578  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2579 
2580  for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
2581  TreeEntry &TE = VectorizableTree[I];
2582 
2583  // We create duplicate tree entries for gather sequences that have multiple
2584  // uses. However, we should not compute the cost of duplicate sequences.
2585  // For example, if we have a build vector (i.e., insertelement sequence)
2586  // that is used by more than one vector instruction, we only need to
2587  // compute the cost of the insertelement instructions once. The redundent
2588  // instructions will be eliminated by CSE.
2589  //
2590  // We should consider not creating duplicate tree entries for gather
2591  // sequences, and instead add additional edges to the tree representing
2592  // their uses. Since such an approach results in fewer total entries,
2593  // existing heuristics based on tree size may yeild different results.
2594  //
2595  if (TE.NeedToGather &&
2596  std::any_of(std::next(VectorizableTree.begin(), I + 1),
2597  VectorizableTree.end(), [TE](TreeEntry &Entry) {
2598  return Entry.NeedToGather && Entry.isSame(TE.Scalars);
2599  }))
2600  continue;
2601 
2602  int C = getEntryCost(&TE);
2603  DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "
2604  << *TE.Scalars[0] << ".\n");
2605  Cost += C;
2606  }
2607 
2608  SmallSet<Value *, 16> ExtractCostCalculated;
2609  int ExtractCost = 0;
2610  for (ExternalUser &EU : ExternalUses) {
2611  // We only add extract cost once for the same scalar.
2612  if (!ExtractCostCalculated.insert(EU.Scalar).second)
2613  continue;
2614 
2615  // Uses by ephemeral values are free (because the ephemeral value will be
2616  // removed prior to code generation, and so the extraction will be
2617  // removed as well).
2618  if (EphValues.count(EU.User))
2619  continue;
2620 
2621  // If we plan to rewrite the tree in a smaller type, we will need to sign
2622  // extend the extracted value back to the original type. Here, we account
2623  // for the extract and the added cost of the sign extend if needed.
2624  auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2625  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2626  if (MinBWs.count(ScalarRoot)) {
2627  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2628  auto Extend =
2629  MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2630  VecTy = VectorType::get(MinTy, BundleWidth);
2631  ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2632  VecTy, EU.Lane);
2633  } else {
2634  ExtractCost +=
2635  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2636  }
2637  }
2638 
2639  int SpillCost = getSpillCost();
2640  Cost += SpillCost + ExtractCost;
2641 
2642  std::string Str;
2643  {
2644  raw_string_ostream OS(Str);
2645  OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2646  << "SLP: Extract Cost = " << ExtractCost << ".\n"
2647  << "SLP: Total Cost = " << Cost << ".\n";
2648  }
2649  DEBUG(dbgs() << Str);
2650 
2651  if (ViewSLPTree)
2652  ViewGraph(this, "SLP" + F->getName(), false, Str);
2653 
2654  return Cost;
2655 }
2656 
2657 int BoUpSLP::getGatherCost(Type *Ty,
2658  const DenseSet<unsigned> &ShuffledIndices) {
2659  int Cost = 0;
2660  for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2661  if (!ShuffledIndices.count(i))
2662  Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2663  if (!ShuffledIndices.empty())
2664  Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2665  return Cost;
2666 }
2667 
2668 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2669  // Find the type of the operands in VL.
2670  Type *ScalarTy = VL[0]->getType();
2671  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2672  ScalarTy = SI->getValueOperand()->getType();
2673  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2674  // Find the cost of inserting/extracting values from the vector.
2675  // Check if the same elements are inserted several times and count them as
2676  // shuffle candidates.
2677  DenseSet<unsigned> ShuffledElements;
2678  DenseSet<Value *> UniqueElements;
2679  // Iterate in reverse order to consider insert elements with the high cost.
2680  for (unsigned I = VL.size(); I > 0; --I) {
2681  unsigned Idx = I - 1;
2682  if (!UniqueElements.insert(VL[Idx]).second)
2683  ShuffledElements.insert(Idx);
2684  }
2685  return getGatherCost(VecTy, ShuffledElements);
2686 }
2687 
2688 // Reorder commutative operations in alternate shuffle if the resulting vectors
2689 // are consecutive loads. This would allow us to vectorize the tree.
2690 // If we have something like-
2691 // load a[0] - load b[0]
2692 // load b[1] + load a[1]
2693 // load a[2] - load b[2]
2694 // load a[3] + load b[3]
2695 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2696 // code.
2697 void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2700  // Push left and right operands of binary operation into Left and Right
2701  unsigned AltOpcode = getAltOpcode(Opcode);
2702  (void)AltOpcode;
2703  for (Value *V : VL) {
2704  auto *I = cast<Instruction>(V);
2705  assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&
2706  "Incorrect instruction in vector");
2707  Left.push_back(I->getOperand(0));
2708  Right.push_back(I->getOperand(1));
2709  }
2710 
2711  // Reorder if we have a commutative operation and consecutive access
2712  // are on either side of the alternate instructions.
2713  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2714  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2715  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2716  Instruction *VL1 = cast<Instruction>(VL[j]);
2717  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2718  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2719  std::swap(Left[j], Right[j]);
2720  continue;
2721  } else if (VL2->isCommutative() &&
2722  isConsecutiveAccess(L, L1, *DL, *SE)) {
2723  std::swap(Left[j + 1], Right[j + 1]);
2724  continue;
2725  }
2726  // else unchanged
2727  }
2728  }
2729  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2730  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2731  Instruction *VL1 = cast<Instruction>(VL[j]);
2732  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2733  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2734  std::swap(Left[j], Right[j]);
2735  continue;
2736  } else if (VL2->isCommutative() &&
2737  isConsecutiveAccess(L, L1, *DL, *SE)) {
2738  std::swap(Left[j + 1], Right[j + 1]);
2739  continue;
2740  }
2741  // else unchanged
2742  }
2743  }
2744  }
2745 }
2746 
2747 // Return true if I should be commuted before adding it's left and right
2748 // operands to the arrays Left and Right.
2749 //
2750 // The vectorizer is trying to either have all elements one side being
2751 // instruction with the same opcode to enable further vectorization, or having
2752 // a splat to lower the vectorizing cost.
2754  int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2755  ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2756  bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2757  VLeft = I.getOperand(0);
2758  VRight = I.getOperand(1);
2759  // If we have "SplatRight", try to see if commuting is needed to preserve it.
2760  if (SplatRight) {
2761  if (VRight == Right[i - 1])
2762  // Preserve SplatRight
2763  return false;
2764  if (VLeft == Right[i - 1]) {
2765  // Commuting would preserve SplatRight, but we don't want to break
2766  // SplatLeft either, i.e. preserve the original order if possible.
2767  // (FIXME: why do we care?)
2768  if (SplatLeft && VLeft == Left[i - 1])
2769  return false;
2770  return true;
2771  }
2772  }
2773  // Symmetrically handle Right side.
2774  if (SplatLeft) {
2775  if (VLeft == Left[i - 1])
2776  // Preserve SplatLeft
2777  return false;
2778  if (VRight == Left[i - 1])
2779  return true;
2780  }
2781 
2782  Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2783  Instruction *IRight = dyn_cast<Instruction>(VRight);
2784 
2785  // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2786  // it and not the right, in this case we want to commute.
2787  if (AllSameOpcodeRight) {
2788  unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2789  if (IRight && RightPrevOpcode == IRight->getOpcode())
2790  // Do not commute, a match on the right preserves AllSameOpcodeRight
2791  return false;
2792  if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2793  // We have a match and may want to commute, but first check if there is
2794  // not also a match on the existing operands on the Left to preserve
2795  // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2796  // (FIXME: why do we care?)
2797  if (AllSameOpcodeLeft && ILeft &&
2798  cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2799  return false;
2800  return true;
2801  }
2802  }
2803  // Symmetrically handle Left side.
2804  if (AllSameOpcodeLeft) {
2805  unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2806  if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2807  return false;
2808  if (IRight && LeftPrevOpcode == IRight->getOpcode())
2809  return true;
2810  }
2811  return false;
2812 }
2813 
2814 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2815  ArrayRef<Value *> VL,
2817  SmallVectorImpl<Value *> &Right) {
2818  if (!VL.empty()) {
2819  // Peel the first iteration out of the loop since there's nothing
2820  // interesting to do anyway and it simplifies the checks in the loop.
2821  auto *I = cast<Instruction>(VL[0]);
2822  Value *VLeft = I->getOperand(0);
2823  Value *VRight = I->getOperand(1);
2824  if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2825  // Favor having instruction to the right. FIXME: why?
2826  std::swap(VLeft, VRight);
2827  Left.push_back(VLeft);
2828  Right.push_back(VRight);
2829  }
2830 
2831  // Keep track if we have instructions with all the same opcode on one side.
2832  bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2833  bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2834  // Keep track if we have one side with all the same value (broadcast).
2835  bool SplatLeft = true;
2836  bool SplatRight = true;
2837 
2838  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2839  Instruction *I = cast<Instruction>(VL[i]);
2840  assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2841  (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2842  "Can only process commutative instruction");
2843  // Commute to favor either a splat or maximizing having the same opcodes on
2844  // one side.
2845  Value *VLeft;
2846  Value *VRight;
2847  if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2848  AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2849  VRight)) {
2850  Left.push_back(VRight);
2851  Right.push_back(VLeft);
2852  } else {
2853  Left.push_back(VLeft);
2854  Right.push_back(VRight);
2855  }
2856  // Update Splat* and AllSameOpcode* after the insertion.
2857  SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2858  SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2859  AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2860  (cast<Instruction>(Left[i - 1])->getOpcode() ==
2861  cast<Instruction>(Left[i])->getOpcode());
2862  AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2863  (cast<Instruction>(Right[i - 1])->getOpcode() ==
2864  cast<Instruction>(Right[i])->getOpcode());
2865  }
2866 
2867  // If one operand end up being broadcast, return this operand order.
2868  if (SplatRight || SplatLeft)
2869  return;
2870 
2871  // Finally check if we can get longer vectorizable chain by reordering
2872  // without breaking the good operand order detected above.
2873  // E.g. If we have something like-
2874  // load a[0] load b[0]
2875  // load b[1] load a[1]
2876  // load a[2] load b[2]
2877  // load a[3] load b[3]
2878  // Reordering the second load b[1] load a[1] would allow us to vectorize
2879  // this code and we still retain AllSameOpcode property.
2880  // FIXME: This load reordering might break AllSameOpcode in some rare cases
2881  // such as-
2882  // add a[0],c[0] load b[0]
2883  // add a[1],c[2] load b[1]
2884  // b[2] load b[2]
2885  // add a[3],c[3] load b[3]
2886  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2887  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2888  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2889  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2890  std::swap(Left[j + 1], Right[j + 1]);
2891  continue;
2892  }
2893  }
2894  }
2895  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2896  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2897  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2898  std::swap(Left[j + 1], Right[j + 1]);
2899  continue;
2900  }
2901  }
2902  }
2903  // else unchanged
2904  }
2905 }
2906 
2907 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2908  // Get the basic block this bundle is in. All instructions in the bundle
2909  // should be in this block.
2910  auto *Front = cast<Instruction>(OpValue);
2911  auto *BB = Front->getParent();
2912  const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2913  const unsigned AltOpcode = getAltOpcode(Opcode);
2914  assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2915  return !sameOpcodeOrAlt(Opcode, AltOpcode,
2916  cast<Instruction>(V)->getOpcode()) ||
2917  cast<Instruction>(V)->getParent() == BB;
2918  }));
2919 
2920  // The last instruction in the bundle in program order.
2921  Instruction *LastInst = nullptr;
2922 
2923  // Find the last instruction. The common case should be that BB has been
2924  // scheduled, and the last instruction is VL.back(). So we start with
2925  // VL.back() and iterate over schedule data until we reach the end of the
2926  // bundle. The end of the bundle is marked by null ScheduleData.
2927  if (BlocksSchedules.count(BB)) {
2928  auto *Bundle =
2929  BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2930  if (Bundle && Bundle->isPartOfBundle())
2931  for (; Bundle; Bundle = Bundle->NextInBundle)
2932  if (Bundle->OpValue == Bundle->Inst)
2933  LastInst = Bundle->Inst;
2934  }
2935 
2936  // LastInst can still be null at this point if there's either not an entry
2937  // for BB in BlocksSchedules or there's no ScheduleData available for
2938  // VL.back(). This can be the case if buildTree_rec aborts for various
2939  // reasons (e.g., the maximum recursion depth is reached, the maximum region
2940  // size is reached, etc.). ScheduleData is initialized in the scheduling
2941  // "dry-run".
2942  //
2943  // If this happens, we can still find the last instruction by brute force. We
2944  // iterate forwards from Front (inclusive) until we either see all
2945  // instructions in the bundle or reach the end of the block. If Front is the
2946  // last instruction in program order, LastInst will be set to Front, and we
2947  // will visit all the remaining instructions in the block.
2948  //
2949  // One of the reasons we exit early from buildTree_rec is to place an upper
2950  // bound on compile-time. Thus, taking an additional compile-time hit here is
2951  // not ideal. However, this should be exceedingly rare since it requires that
2952  // we both exit early from buildTree_rec and that the bundle be out-of-order
2953  // (causing us to iterate all the way to the end of the block).
2954  if (!LastInst) {
2955  SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2956  for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2957  if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2958  LastInst = &I;
2959  if (Bundle.empty())
2960  break;
2961  }
2962  }
2963 
2964  // Set the insertion point after the last instruction in the bundle. Set the
2965  // debug location to Front.
2966  Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2967  Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2968 }
2969 
2970 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2971  Value *Vec = UndefValue::get(Ty);
2972  // Generate the 'InsertElement' instruction.
2973  for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2974  Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2975  if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2976  GatherSeq.insert(Insrt);
2977  CSEBlocks.insert(Insrt->getParent());
2978 
2979  // Add to our 'need-to-extract' list.
2980  if (TreeEntry *E = getTreeEntry(VL[i])) {
2981  // Find which lane we need to extract.
2982  int FoundLane = -1;
2983  for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2984  // Is this the lane of the scalar that we are looking for ?
2985  if (E->Scalars[Lane] == VL[i]) {
2986  FoundLane = Lane;
2987  break;
2988  }
2989  }
2990  assert(FoundLane >= 0 && "Could not find the correct lane");
2991  if (!E->ReuseShuffleIndices.empty()) {
2992  FoundLane =
2993  std::distance(E->ReuseShuffleIndices.begin(),
2994  llvm::find(E->ReuseShuffleIndices, FoundLane));
2995  }
2996  ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2997  }
2998  }
2999  }
3000 
3001  return Vec;
3002 }
3003 
3004 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
3005  InstructionsState S = getSameOpcode(VL);
3006  if (S.Opcode) {
3007  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
3008  if (E->isSame(VL)) {
3009  Value *V = vectorizeTree(E);
3010  if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
3011  // We need to get the vectorized value but without shuffle.
3012  if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
3013  V = SV->getOperand(0);
3014  } else {
3015  // Reshuffle to get only unique values.
3016  SmallVector<unsigned, 4> UniqueIdxs;
3017  SmallSet<unsigned, 4> UsedIdxs;
3018  for(unsigned Idx : E->ReuseShuffleIndices)
3019  if (UsedIdxs.insert(Idx).second)
3020  UniqueIdxs.emplace_back(Idx);
3021  V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3022  UniqueIdxs);
3023  }
3024  }
3025  return V;
3026  }
3027  }
3028  }
3029 
3030  Type *ScalarTy = S.OpValue->getType();
3031  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
3032  ScalarTy = SI->getValueOperand()->getType();
3033 
3034  // Check that every instruction appears once in this bundle.
3035  SmallVector<unsigned, 4> ReuseShuffleIndicies;
3036  SmallVector<Value *, 4> UniqueValues;
3037  if (VL.size() > 2) {
3038  DenseMap<Value *, unsigned> UniquePositions;
3039  for (Value *V : VL) {
3040  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
3041  ReuseShuffleIndicies.emplace_back(Res.first->second);
3042  if (Res.second || isa<Constant>(V))
3043  UniqueValues.emplace_back(V);
3044  }
3045  // Do not shuffle single element or if number of unique values is not power
3046  // of 2.
3047  if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
3048  !llvm::isPowerOf2_32(UniqueValues.size()))
3049  ReuseShuffleIndicies.clear();
3050  else
3051  VL = UniqueValues;
3052  }
3053  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3054 
3055  Value *V = Gather(VL, VecTy);
3056  if (!ReuseShuffleIndicies.empty()) {
3057  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3058  ReuseShuffleIndicies, "shuffle");
3059  if (auto *I = dyn_cast<Instruction>(V)) {
3060  GatherSeq.insert(I);
3061  CSEBlocks.insert(I->getParent());
3062  }
3063  }
3064  return V;
3065 }
3066 
3069  Mask.clear();
3070  const unsigned E = Indices.size();
3071  Mask.resize(E);
3072  for (unsigned I = 0; I < E; ++I)
3073  Mask[Indices[I]] = I;
3074 }
3075 
3076 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
3077  IRBuilder<>::InsertPointGuard Guard(Builder);
3078 
3079  if (E->VectorizedValue) {
3080  DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
3081  return E->VectorizedValue;
3082  }
3083 
3084  InstructionsState S = getSameOpcode(E->Scalars);
3085  Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
3086  Type *ScalarTy = VL0->getType();
3087  if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3088  ScalarTy = SI->getValueOperand()->getType();
3089  VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
3090 
3091  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3092 
3093  if (E->NeedToGather) {
3094  setInsertPointAfterBundle(E->Scalars, VL0);
3095  auto *V = Gather(E->Scalars, VecTy);
3096  if (NeedToShuffleReuses) {
3097  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3098  E->ReuseShuffleIndices, "shuffle");
3099  if (auto *I = dyn_cast<Instruction>(V)) {
3100  GatherSeq.insert(I);
3101  CSEBlocks.insert(I->getParent());
3102  }
3103  }
3104  E->VectorizedValue = V;
3105  return V;
3106  }
3107 
3108  unsigned ShuffleOrOp = S.IsAltShuffle ?
3109  (unsigned) Instruction::ShuffleVector : S.Opcode;
3110  switch (ShuffleOrOp) {
3111  case Instruction::PHI: {
3112  PHINode *PH = dyn_cast<PHINode>(VL0);
3113  Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
3114  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3115  PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
3116  Value *V = NewPhi;
3117  if (NeedToShuffleReuses) {
3118  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3119  E->ReuseShuffleIndices, "shuffle");
3120  }
3121  E->VectorizedValue = V;
3122 
3123  // PHINodes may have multiple entries from the same block. We want to
3124  // visit every block once.
3125  SmallSet<BasicBlock*, 4> VisitedBBs;
3126 
3127  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
3128  ValueList Operands;
3129  BasicBlock *IBB = PH->getIncomingBlock(i);
3130 
3131  if (!VisitedBBs.insert(IBB).second) {
3132  NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
3133  continue;
3134  }
3135 
3136  // Prepare the operand vector.
3137  for (Value *V : E->Scalars)
3138  Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
3139 
3140  Builder.SetInsertPoint(IBB->getTerminator());
3141  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3142  Value *Vec = vectorizeTree(Operands);
3143  NewPhi->addIncoming(Vec, IBB);
3144  }
3145 
3146  assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
3147  "Invalid number of incoming values");
3148  return V;
3149  }
3150 
3151  case Instruction::ExtractElement: {
3152  if (!E->NeedToGather) {
3153  Value *V = VL0->getOperand(0);
3154  if (!E->ReorderIndices.empty()) {
3155  OrdersType Mask;
3156  inversePermutation(E->ReorderIndices, Mask);
3157  Builder.SetInsertPoint(VL0);
3158  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
3159  "reorder_shuffle");
3160  }
3161  if (NeedToShuffleReuses) {
3162  // TODO: Merge this shuffle with the ReorderShuffleMask.
3163  if (!E->ReorderIndices.empty())
3164  Builder.SetInsertPoint(VL0);
3165  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3166  E->ReuseShuffleIndices, "shuffle");
3167  }
3168  E->VectorizedValue = V;
3169  return V;
3170  }
3171  setInsertPointAfterBundle(E->Scalars, VL0);
3172  auto *V = Gather(E->Scalars, VecTy);
3173  if (NeedToShuffleReuses) {
3174  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3175  E->ReuseShuffleIndices, "shuffle");
3176  if (auto *I = dyn_cast<Instruction>(V)) {
3177  GatherSeq.insert(I);
3178  CSEBlocks.insert(I->getParent());
3179  }
3180  }
3181  E->VectorizedValue = V;
3182  return V;
3183  }
3184  case Instruction::ExtractValue: {
3185  if (!E->NeedToGather) {
3186  LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3187  Builder.SetInsertPoint(LI);
3188  PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3189  Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3190  LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
3191  Value *NewV = propagateMetadata(V, E->Scalars);
3192  if (!E->ReorderIndices.empty()) {
3193  OrdersType Mask;
3194  inversePermutation(E->ReorderIndices, Mask);
3195  NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
3196  "reorder_shuffle");
3197  }
3198  if (NeedToShuffleReuses) {
3199  // TODO: Merge this shuffle with the ReorderShuffleMask.
3200  NewV = Builder.CreateShuffleVector(
3201  NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3202  }
3203  E->VectorizedValue = NewV;
3204  return NewV;
3205  }
3206  setInsertPointAfterBundle(E->Scalars, VL0);
3207  auto *V = Gather(E->Scalars, VecTy);
3208  if (NeedToShuffleReuses) {
3209  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3210  E->ReuseShuffleIndices, "shuffle");
3211  if (auto *I = dyn_cast<Instruction>(V)) {
3212  GatherSeq.insert(I);
3213  CSEBlocks.insert(I->getParent());
3214  }
3215  }
3216  E->VectorizedValue = V;
3217  return V;
3218  }
3219  case Instruction::ZExt:
3220  case Instruction::SExt:
3221  case Instruction::FPToUI:
3222  case Instruction::FPToSI:
3223  case Instruction::FPExt:
3224  case Instruction::PtrToInt:
3225  case Instruction::IntToPtr:
3226  case Instruction::SIToFP:
3227  case Instruction::UIToFP:
3228  case Instruction::Trunc:
3229  case Instruction::FPTrunc:
3230  case Instruction::BitCast: {
3231  ValueList INVL;
3232  for (Value *V : E->Scalars)
3233  INVL.push_back(cast<Instruction>(V)->getOperand(0));
3234 
3235  setInsertPointAfterBundle(E->Scalars, VL0);
3236 
3237  Value *InVec = vectorizeTree(INVL);
3238 
3239  if (E->VectorizedValue) {
3240  DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3241  return E->VectorizedValue;
3242  }
3243 
3244  CastInst *CI = dyn_cast<CastInst>(VL0);
3245  Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3246  if (NeedToShuffleReuses) {
3247  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3248  E->ReuseShuffleIndices, "shuffle");
3249  }
3250  E->VectorizedValue = V;
3251  ++NumVectorInstructions;
3252  return V;
3253  }
3254  case Instruction::FCmp:
3255  case Instruction::ICmp: {
3256  ValueList LHSV, RHSV;
3257  for (Value *V : E->Scalars) {
3258  LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3259  RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3260  }
3261 
3262  setInsertPointAfterBundle(E->Scalars, VL0);
3263 
3264  Value *L = vectorizeTree(LHSV);
3265  Value *R = vectorizeTree(RHSV);
3266 
3267  if (E->VectorizedValue) {
3268  DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3269  return E->VectorizedValue;
3270  }
3271 
3272  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3273  Value *V;
3274  if (S.Opcode == Instruction::FCmp)
3275  V = Builder.CreateFCmp(P0, L, R);
3276  else
3277  V = Builder.CreateICmp(P0, L, R);
3278 
3279  propagateIRFlags(V, E->Scalars, VL0);
3280  if (NeedToShuffleReuses) {
3281  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3282  E->ReuseShuffleIndices, "shuffle");
3283  }
3284  E->VectorizedValue = V;
3285  ++NumVectorInstructions;
3286  return V;
3287  }
3288  case Instruction::Select: {
3289  ValueList TrueVec, FalseVec, CondVec;
3290  for (Value *V : E->Scalars) {
3291  CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3292  TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3293  FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3294  }
3295 
3296  setInsertPointAfterBundle(E->Scalars, VL0);
3297 
3298  Value *Cond = vectorizeTree(CondVec);
3299  Value *True = vectorizeTree(TrueVec);
3300  Value *False = vectorizeTree(FalseVec);
3301 
3302  if (E->VectorizedValue) {
3303  DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3304  return E->VectorizedValue;
3305  }
3306 
3307  Value *V = Builder.CreateSelect(Cond, True, False);
3308  if (NeedToShuffleReuses) {
3309  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3310  E->ReuseShuffleIndices, "shuffle");
3311  }
3312  E->VectorizedValue = V;
3313  ++NumVectorInstructions;
3314  return V;
3315  }
3316  case Instruction::Add:
3317  case Instruction::FAdd:
3318  case Instruction::Sub:
3319  case Instruction::FSub:
3320  case Instruction::Mul:
3321  case Instruction::FMul:
3322  case Instruction::UDiv:
3323  case Instruction::SDiv:
3324  case Instruction::FDiv:
3325  case Instruction::URem:
3326  case Instruction::SRem:
3327  case Instruction::FRem:
3328  case Instruction::Shl:
3329  case Instruction::LShr:
3330  case Instruction::AShr:
3331  case Instruction::And:
3332  case Instruction::Or:
3333  case Instruction::Xor: {
3334  ValueList LHSVL, RHSVL;
3335  if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3336  reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
3337  RHSVL);
3338  else
3339  for (Value *V : E->Scalars) {
3340  auto *I = cast<Instruction>(V);
3341  LHSVL.push_back(I->getOperand(0));
3342  RHSVL.push_back(I->getOperand(1));
3343  }
3344 
3345  setInsertPointAfterBundle(E->Scalars, VL0);
3346 
3347  Value *LHS = vectorizeTree(LHSVL);
3348  Value *RHS = vectorizeTree(RHSVL);
3349 
3350  if (E->VectorizedValue) {
3351  DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3352  return E->VectorizedValue;
3353  }
3354 
3355  Value *V = Builder.CreateBinOp(
3356  static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3357  propagateIRFlags(V, E->Scalars, VL0);
3358  if (auto *I = dyn_cast<Instruction>(V))
3359  V = propagateMetadata(I, E->Scalars);
3360 
3361  if (NeedToShuffleReuses) {
3362  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3363  E->ReuseShuffleIndices, "shuffle");
3364  }
3365  E->VectorizedValue = V;
3366  ++NumVectorInstructions;
3367 
3368  return V;
3369  }
3370  case Instruction::Load: {
3371  // Loads are inserted at the head of the tree because we don't want to
3372  // sink them all the way down past store instructions.
3373  bool IsReorder = !E->ReorderIndices.empty();
3374  if (IsReorder)
3375  VL0 = cast<Instruction>(E->Scalars[E->ReorderIndices.front()]);
3376  setInsertPointAfterBundle(E->Scalars, VL0);
3377 
3378  LoadInst *LI = cast<LoadInst>(VL0);
3379  Type *ScalarLoadTy = LI->getType();
3380  unsigned AS = LI->getPointerAddressSpace();
3381 
3382  Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3383  VecTy->getPointerTo(AS));
3384 
3385  // The pointer operand uses an in-tree scalar so we add the new BitCast to
3386  // ExternalUses list to make sure that an extract will be generated in the
3387  // future.
3388  Value *PO = LI->getPointerOperand();
3389  if (getTreeEntry(PO))
3390  ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3391 
3392  unsigned Alignment = LI->getAlignment();
3393  LI = Builder.CreateLoad(VecPtr);
3394  if (!Alignment) {
3395  Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3396  }
3397  LI->setAlignment(Alignment);
3398  Value *V = propagateMetadata(LI, E->Scalars);
3399  if (IsReorder) {
3400  OrdersType Mask;
3401  inversePermutation(E->ReorderIndices, Mask);
3402  V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3403  Mask, "reorder_shuffle");
3404  }
3405  if (NeedToShuffleReuses) {
3406  // TODO: Merge this shuffle with the ReorderShuffleMask.
3407  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3408  E->ReuseShuffleIndices, "shuffle");
3409  }
3410  E->VectorizedValue = V;
3411  ++NumVectorInstructions;
3412  return V;
3413  }
3414  case Instruction::Store: {
3415  StoreInst *SI = cast<StoreInst>(VL0);
3416  unsigned Alignment = SI->getAlignment();
3417  unsigned AS = SI->getPointerAddressSpace();
3418 
3419  ValueList ScalarStoreValues;
3420  for (Value *V : E->Scalars)
3421  ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3422 
3423  setInsertPointAfterBundle(E->Scalars, VL0);
3424 
3425  Value *VecValue = vectorizeTree(ScalarStoreValues);
3426  Value *ScalarPtr = SI->getPointerOperand();
3427  Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3428  StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3429 
3430  // The pointer operand uses an in-tree scalar, so add the new BitCast to
3431  // ExternalUses to make sure that an extract will be generated in the
3432  // future.
3433  if (getTreeEntry(ScalarPtr))
3434  ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3435 
3436  if (!Alignment)
3437  Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3438 
3439  S->setAlignment(Alignment);
3440  Value *V = propagateMetadata(S, E->Scalars);
3441  if (NeedToShuffleReuses) {
3442  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3443  E->ReuseShuffleIndices, "shuffle");
3444  }
3445  E->VectorizedValue = V;
3446  ++NumVectorInstructions;
3447  return V;
3448  }
3449  case Instruction::GetElementPtr: {
3450  setInsertPointAfterBundle(E->Scalars, VL0);
3451 
3452  ValueList Op0VL;
3453  for (Value *V : E->Scalars)
3454  Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3455 
3456  Value *Op0 = vectorizeTree(Op0VL);
3457 
3458  std::vector<Value *> OpVecs;
3459  for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3460  ++j) {
3461  ValueList OpVL;
3462  for (Value *V : E->Scalars)
3463  OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3464 
3465  Value *OpVec = vectorizeTree(OpVL);
3466  OpVecs.push_back(OpVec);
3467  }
3468 
3469  Value *V = Builder.CreateGEP(
3470  cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3471  if (Instruction *I = dyn_cast<Instruction>(V))
3472  V = propagateMetadata(I, E->Scalars);
3473 
3474  if (NeedToShuffleReuses) {
3475  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3476  E->ReuseShuffleIndices, "shuffle");
3477  }
3478  E->VectorizedValue = V;
3479  ++NumVectorInstructions;
3480 
3481  return V;
3482  }
3483  case Instruction::Call: {
3484  CallInst *CI = cast<CallInst>(VL0);
3485  setInsertPointAfterBundle(E->Scalars, VL0);
3486  Function *FI;
3488  Value *ScalarArg = nullptr;
3489  if (CI && (FI = CI->getCalledFunction())) {
3490  IID = FI->getIntrinsicID();
3491  }
3492  std::vector<Value *> OpVecs;
3493  for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3494  ValueList OpVL;
3495  // ctlz,cttz and powi are special intrinsics whose second argument is
3496  // a scalar. This argument should not be vectorized.
3497  if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3498  CallInst *CEI = cast<CallInst>(VL0);
3499  ScalarArg = CEI->getArgOperand(j);
3500  OpVecs.push_back(CEI->getArgOperand(j));
3501  continue;
3502  }
3503  for (Value *V : E->Scalars) {
3504  CallInst *CEI = cast<CallInst>(V);
3505  OpVL.push_back(CEI->getArgOperand(j));
3506  }
3507 
3508  Value *OpVec = vectorizeTree(OpVL);
3509  DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3510  OpVecs.push_back(OpVec);
3511  }
3512 
3513  Module *M = F->getParent();
3515  Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3516  Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3518  CI->getOperandBundlesAsDefs(OpBundles);
3519  Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3520 
3521  // The scalar argument uses an in-tree scalar so we add the new vectorized
3522  // call to ExternalUses list to make sure that an extract will be
3523  // generated in the future.
3524  if (ScalarArg && getTreeEntry(ScalarArg))
3525  ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3526 
3527  propagateIRFlags(V, E->Scalars, VL0);
3528  if (NeedToShuffleReuses) {
3529  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3530  E->ReuseShuffleIndices, "shuffle");
3531  }
3532  E->VectorizedValue = V;
3533  ++NumVectorInstructions;
3534  return V;
3535  }
3536  case Instruction::ShuffleVector: {
3537  ValueList LHSVL, RHSVL;
3538  assert(Instruction::isBinaryOp(S.Opcode) &&
3539  "Invalid Shuffle Vector Operand");
3540  reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3541  setInsertPointAfterBundle(E->Scalars, VL0);
3542 
3543  Value *LHS = vectorizeTree(LHSVL);
3544  Value *RHS = vectorizeTree(RHSVL);
3545 
3546  if (E->VectorizedValue) {
3547  DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3548  return E->VectorizedValue;
3549  }
3550 
3551  // Create a vector of LHS op1 RHS
3552  Value *V0 = Builder.CreateBinOp(
3553  static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3554 
3555  unsigned AltOpcode = getAltOpcode(S.Opcode);
3556  // Create a vector of LHS op2 RHS
3557  Value *V1 = Builder.CreateBinOp(
3558  static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3559 
3560  // Create shuffle to take alternate operations from the vector.
3561  // Also, gather up odd and even scalar ops to propagate IR flags to
3562  // each vector operation.
3563  ValueList OddScalars, EvenScalars;
3564  unsigned e = E->Scalars.size();
3566  for (unsigned i = 0; i < e; ++i) {
3567  if (isOdd(i)) {
3568  Mask[i] = Builder.getInt32(e + i);
3569  OddScalars.push_back(E->Scalars[i]);
3570  } else {
3571  Mask[i] = Builder.getInt32(i);
3572  EvenScalars.push_back(E->Scalars[i]);
3573  }
3574  }
3575 
3576  Value *ShuffleMask = ConstantVector::get(Mask);
3577  propagateIRFlags(V0, EvenScalars);
3578  propagateIRFlags(V1, OddScalars);
3579 
3580  Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3581  if (Instruction *I = dyn_cast<Instruction>(V))
3582  V = propagateMetadata(I, E->Scalars);
3583  if (NeedToShuffleReuses) {
3584  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3585  E->ReuseShuffleIndices, "shuffle");
3586  }
3587  E->VectorizedValue = V;
3588  ++NumVectorInstructions;
3589 
3590  return V;
3591  }
3592  default:
3593  llvm_unreachable("unknown inst");
3594  }
3595  return nullptr;
3596 }
3597 
3598 Value *BoUpSLP::vectorizeTree() {
3599  ExtraValueToDebugLocsMap ExternallyUsedValues;
3600  return vectorizeTree(ExternallyUsedValues);
3601 }
3602 
3603 Value *
3604 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3605  // All blocks must be scheduled before any instructions are inserted.
3606  for (auto &BSIter : BlocksSchedules) {
3607  scheduleBlock(BSIter.second.get());
3608  }
3609 
3610  Builder.SetInsertPoint(&F->getEntryBlock().front());
3611  auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3612 
3613  // If the vectorized tree can be rewritten in a smaller type, we truncate the
3614  // vectorized root. InstCombine will then rewrite the entire expression. We
3615  // sign extend the extracted values below.
3616  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3617  if (MinBWs.count(ScalarRoot)) {
3618  if (auto *I = dyn_cast<Instruction>(VectorRoot))
3619  Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3620  auto BundleWidth = VectorizableTree[0].Scalars.size();
3621  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3622  auto *VecTy = VectorType::get(MinTy, BundleWidth);
3623  auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3624  VectorizableTree[0].VectorizedValue = Trunc;
3625  }
3626 
3627  DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n");
3628 
3629  // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3630  // specified by ScalarType.
3631  auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3632  if (!MinBWs.count(ScalarRoot))
3633  return Ex;
3634  if (MinBWs[ScalarRoot].second)
3635  return Builder.CreateSExt(Ex, ScalarType);
3636  return Builder.CreateZExt(Ex, ScalarType);
3637  };
3638 
3639  // Extract all of the elements with the external uses.
3640  for (const auto &ExternalUse : ExternalUses) {
3641  Value *Scalar = ExternalUse.Scalar;
3642  llvm::User *User = ExternalUse.User;
3643 
3644  // Skip users that we already RAUW. This happens when one instruction
3645  // has multiple uses of the same value.
3646  if (User && !is_contained(Scalar->users(), User))
3647  continue;
3648  TreeEntry *E = getTreeEntry(Scalar);
3649  assert(E && "Invalid scalar");
3650  assert(!E->NeedToGather && "Extracting from a gather list");
3651 
3652  Value *Vec = E->VectorizedValue;
3653  assert(Vec && "Can't find vectorizable value");
3654 
3655  Value *Lane = Builder.getInt32(ExternalUse.Lane);
3656  // If User == nullptr, the Scalar is used as extra arg. Generate
3657  // ExtractElement instruction and update the record for this scalar in
3658  // ExternallyUsedValues.
3659  if (!User) {
3660  assert(ExternallyUsedValues.count(Scalar) &&
3661  "Scalar with nullptr as an external user must be registered in "
3662  "ExternallyUsedValues map");
3663  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3664  Builder.SetInsertPoint(VecI->getParent(),
3665  std::next(VecI->getIterator()));
3666  } else {
3667  Builder.SetInsertPoint(&F->getEntryBlock().front());
3668  }
3669  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3670  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3671  CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3672  auto &Locs = ExternallyUsedValues[Scalar];
3673  ExternallyUsedValues.insert({Ex, Locs});
3674  ExternallyUsedValues.erase(Scalar);
3675  continue;
3676  }
3677 
3678  // Generate extracts for out-of-tree users.
3679  // Find the insertion point for the extractelement lane.
3680  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3681  if (PHINode *PH = dyn_cast<PHINode>(User)) {
3682  for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3683  if (PH->getIncomingValue(i) == Scalar) {
3684  TerminatorInst *IncomingTerminator =
3685  PH->getIncomingBlock(i)->getTerminator();
3686  if (isa<CatchSwitchInst>(IncomingTerminator)) {
3687  Builder.SetInsertPoint(VecI->getParent(),
3688  std::next(VecI->getIterator()));
3689  } else {
3690  Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3691  }
3692  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3693  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3694  CSEBlocks.insert(PH->getIncomingBlock(i));
3695  PH->setOperand(i, Ex);
3696  }
3697  }
3698  } else {
3699  Builder.SetInsertPoint(cast<Instruction>(User));
3700  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3701  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3702  CSEBlocks.insert(cast<Instruction>(User)->getParent());
3703  User->replaceUsesOfWith(Scalar, Ex);
3704  }
3705  } else {
3706  Builder.SetInsertPoint(&F->getEntryBlock().front());
3707  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3708  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3709  CSEBlocks.insert(&F->getEntryBlock());
3710  User->replaceUsesOfWith(Scalar, Ex);
3711  }
3712 
3713  DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3714  }
3715 
3716  // For each vectorized value:
3717  for (TreeEntry &EIdx : VectorizableTree) {
3718  TreeEntry *Entry = &EIdx;
3719 
3720  // No need to handle users of gathered values.
3721  if (Entry->NeedToGather)
3722  continue;
3723 
3724  assert(Entry->VectorizedValue && "Can't find vectorizable value");
3725 
3726  // For each lane:
3727  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3728  Value *Scalar = Entry->Scalars[Lane];
3729 
3730  Type *Ty = Scalar->getType();
3731  if (!Ty->isVoidTy()) {
3732 #ifndef NDEBUG
3733  for (User *U : Scalar->users()) {
3734  DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3735 
3736  // It is legal to replace users in the ignorelist by undef.
3737  assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3738  "Replacing out-of-tree value with undef");
3739  }
3740 #endif
3741  Value *Undef = UndefValue::get(Ty);
3742  Scalar->replaceAllUsesWith(Undef);
3743  }
3744  DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3745  eraseInstruction(cast<Instruction>(Scalar));
3746  }
3747  }
3748 
3749  Builder.ClearInsertionPoint();
3750 
3751  return VectorizableTree[0].VectorizedValue;
3752 }
3753 
3754 void BoUpSLP::optimizeGatherSequence() {
3755  DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3756  << " gather sequences instructions.\n");
3757  // LICM InsertElementInst sequences.
3758  for (Instruction *I : GatherSeq) {
3759  if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3760  continue;
3761 
3762  // Check if this block is inside a loop.
3763  Loop *L = LI->getLoopFor(I->getParent());
3764  if (!L)
3765  continue;
3766 
3767  // Check if it has a preheader.
3768  BasicBlock *PreHeader = L->getLoopPreheader();
3769  if (!PreHeader)
3770  continue;
3771 
3772  // If the vector or the element that we insert into it are
3773  // instructions that are defined in this basic block then we can't
3774  // hoist this instruction.
3775  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3776  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3777  if (Op0 && L->contains(Op0))
3778  continue;
3779  if (Op1 && L->contains(Op1))
3780  continue;
3781 
3782  // We can hoist this instruction. Move it to the pre-header.
3783  I->moveBefore(PreHeader->getTerminator());
3784  }
3785 
3786  // Make a list of all reachable blocks in our CSE queue.
3788  CSEWorkList.reserve(CSEBlocks.size());
3789  for (BasicBlock *BB : CSEBlocks)
3790  if (DomTreeNode *N = DT->getNode(BB)) {
3791  assert(DT->isReachableFromEntry(N));
3792  CSEWorkList.push_back(N);
3793  }
3794 
3795  // Sort blocks by domination. This ensures we visit a block after all blocks
3796  // dominating it are visited.
3797  std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3798  [this](const DomTreeNode *A, const DomTreeNode *B) {
3799  return DT->properlyDominates(A, B);
3800  });
3801 
3802  // Perform O(N^2) search over the gather sequences and merge identical
3803  // instructions. TODO: We can further optimize this scan if we split the
3804  // instructions into different buckets based on the insert lane.
3806  for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3807  assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
3808  "Worklist not sorted properly!");
3809  BasicBlock *BB = (*I)->getBlock();
3810  // For all instructions in blocks containing gather sequences:
3811  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3812  Instruction *In = &*it++;
3813  if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3814  continue;
3815 
3816  // Check if we can replace this instruction with any of the
3817  // visited instructions.
3818  for (Instruction *v : Visited) {
3819  if (In->isIdenticalTo(v) &&
3820  DT->dominates(v->getParent(), In->getParent())) {
3821  In->replaceAllUsesWith(v);
3822  eraseInstruction(In);
3823  In = nullptr;
3824  break;
3825  }
3826  }
3827  if (In) {
3828  assert(!is_contained(Visited, In));
3829  Visited.push_back(In);
3830  }
3831  }
3832  }
3833  CSEBlocks.clear();
3834  GatherSeq.clear();
3835 }
3836 
3837 // Groups the instructions to a bundle (which is then a single scheduling entity)
3838 // and schedules instructions until the bundle gets ready.
3839 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3840  BoUpSLP *SLP, Value *OpValue) {
3841  if (isa<PHINode>(OpValue))
3842  return true;
3843 
3844  // Initialize the instruction bundle.
3845  Instruction *OldScheduleEnd = ScheduleEnd;
3846  ScheduleData *PrevInBundle = nullptr;
3847  ScheduleData *Bundle = nullptr;
3848  bool ReSchedule = false;
3849  DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n");
3850 
3851  // Make sure that the scheduling region contains all
3852  // instructions of the bundle.
3853  for (Value *V : VL) {
3854  if (!extendSchedulingRegion(V, OpValue))
3855  return false;
3856  }
3857 
3858  for (Value *V : VL) {
3859  ScheduleData *BundleMember = getScheduleData(V);
3860  assert(BundleMember &&
3861  "no ScheduleData for bundle member (maybe not in same basic block)");
3862  if (BundleMember->IsScheduled) {
3863  // A bundle member was scheduled as single instruction before and now
3864  // needs to be scheduled as part of the bundle. We just get rid of the
3865  // existing schedule.
3866  DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3867  << " was already scheduled\n");
3868  ReSchedule = true;
3869  }
3870  assert(BundleMember->isSchedulingEntity() &&
3871  "bundle member already part of other bundle");
3872  if (PrevInBundle) {
3873  PrevInBundle->NextInBundle = BundleMember;
3874  } else {
3875  Bundle = BundleMember;
3876  }
3877  BundleMember->UnscheduledDepsInBundle = 0;
3878  Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3879 
3880  // Group the instructions to a bundle.
3881  BundleMember->FirstInBundle = Bundle;
3882  PrevInBundle = BundleMember;
3883  }
3884  if (ScheduleEnd != OldScheduleEnd) {
3885  // The scheduling region got new instructions at the lower end (or it is a
3886  // new region for the first bundle). This makes it necessary to
3887  // recalculate all dependencies.
3888  // It is seldom that this needs to be done a second time after adding the
3889  // initial bundle to the region.
3890  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3891  doForAllOpcodes(I, [](ScheduleData *SD) {
3892  SD->clearDependencies();
3893  });
3894  }
3895  ReSchedule = true;
3896  }
3897  if (ReSchedule) {
3898  resetSchedule();
3899  initialFillReadyList(ReadyInsts);
3900  }
3901 
3902  DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3903  << BB->getName() << "\n");
3904 
3905  calculateDependencies(Bundle, true, SLP);
3906 
3907  // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3908  // means that there are no cyclic dependencies and we can schedule it.
3909  // Note that's important that we don't "schedule" the bundle yet (see
3910  // cancelScheduling).
3911  while (!Bundle->isReady() && !ReadyInsts.empty()) {
3912 
3913  ScheduleData *pickedSD = ReadyInsts.back();
3914  ReadyInsts.pop_back();
3915 
3916  if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3917  schedule(pickedSD, ReadyInsts);
3918  }
3919  }
3920  if (!Bundle->isReady()) {
3921  cancelScheduling(VL, OpValue);
3922  return false;
3923  }
3924  return true;
3925 }
3926 
3927 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3928  Value *OpValue) {
3929  if (isa<PHINode>(OpValue))
3930  return;
3931 
3932  ScheduleData *Bundle = getScheduleData(OpValue);
3933  DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3934  assert(!Bundle->IsScheduled &&
3935  "Can't cancel bundle which is already scheduled");
3936  assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3937  "tried to unbundle something which is not a bundle");
3938 
3939  // Un-bundle: make single instructions out of the bundle.
3940  ScheduleData *BundleMember = Bundle;
3941  while (BundleMember) {
3942  assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3943  BundleMember->FirstInBundle = BundleMember;
3944  ScheduleData *Next = BundleMember->NextInBundle;
3945  BundleMember->NextInBundle = nullptr;
3946  BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3947  if (BundleMember->UnscheduledDepsInBundle == 0) {
3948  ReadyInsts.insert(BundleMember);
3949  }
3950  BundleMember = Next;
3951  }
3952 }
3953 
3954 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3955  // Allocate a new ScheduleData for the instruction.
3956  if (ChunkPos >= ChunkSize) {
3957  ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3958  ChunkPos = 0;
3959  }
3960  return &(ScheduleDataChunks.back()[ChunkPos++]);
3961 }
3962 
3963 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3964  Value *OpValue) {
3965  if (getScheduleData(V, isOneOf(OpValue, V)))
3966  return true;
3967  Instruction *I = dyn_cast<Instruction>(V);
3968  assert(I && "bundle member must be an instruction");
3969  assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3970  auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3971  ScheduleData *ISD = getScheduleData(I);
3972  if (!ISD)
3973  return false;
3974  assert(isInSchedulingRegion(ISD) &&
3975  "ScheduleData not in scheduling region");
3976  ScheduleData *SD = allocateScheduleDataChunks();
3977  SD->Inst = I;
3978  SD->init(SchedulingRegionID, OpValue);
3979  ExtraScheduleDataMap[I][OpValue] = SD;
3980  return true;
3981  };
3982  if (CheckSheduleForI(I))
3983  return true;
3984  if (!ScheduleStart) {
3985  // It's the first instruction in the new region.
3986  initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3987  ScheduleStart = I;
3988  ScheduleEnd = I->getNextNode();
3989  if (isOneOf(OpValue, I) != I)
3990  CheckSheduleForI(I);
3991  assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3992  DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3993  return true;
3994  }
3995  // Search up and down at the same time, because we don't know if the new
3996  // instruction is above or below the existing scheduling region.
3998  ++ScheduleStart->getIterator().getReverse();
3999  BasicBlock::reverse_iterator UpperEnd = BB->rend();
4000  BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
4001  BasicBlock::iterator LowerEnd = BB->end();
4002  while (true) {
4003  if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
4004  DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
4005  return false;
4006  }
4007 
4008  if (UpIter != UpperEnd) {
4009  if (&*UpIter == I) {
4010  initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
4011  ScheduleStart = I;
4012  if (isOneOf(OpValue, I) != I)
4013  CheckSheduleForI(I);
4014  DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n");
4015  return true;
4016  }
4017  UpIter++;
4018  }
4019  if (DownIter != LowerEnd) {
4020  if (&*DownIter == I) {
4021  initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
4022  nullptr);
4023  ScheduleEnd = I->getNextNode();
4024  if (isOneOf(OpValue, I) != I)
4025  CheckSheduleForI(I);
4026  assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
4027  DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
4028  return true;
4029  }
4030  DownIter++;
4031  }
4032  assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
4033  "instruction not found in block");
4034  }
4035  return true;
4036 }
4037 
4038 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
4039  Instruction *ToI,
4040  ScheduleData *PrevLoadStore,
4041  ScheduleData *NextLoadStore) {
4042  ScheduleData *CurrentLoadStore = PrevLoadStore;
4043  for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
4044  ScheduleData *SD = ScheduleDataMap[I];
4045  if (!SD) {
4046  SD = allocateScheduleDataChunks();
4047  ScheduleDataMap[I] = SD;
4048  SD->Inst = I;
4049  }
4050  assert(!isInSchedulingRegion(SD) &&
4051  "new ScheduleData already in scheduling region");
4052  SD->init(SchedulingRegionID, I);
4053 
4054  if (I->mayReadOrWriteMemory() &&
4055  (!isa<IntrinsicInst>(I) ||
4056  cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
4057  // Update the linked list of memory accessing instructions.
4058  if (CurrentLoadStore) {
4059  CurrentLoadStore->NextLoadStore = SD;
4060  } else {
4061  FirstLoadStoreInRegion = SD;
4062  }
4063  CurrentLoadStore = SD;
4064  }
4065  }
4066  if (NextLoadStore) {
4067  if (CurrentLoadStore)
4068  CurrentLoadStore->NextLoadStore = NextLoadStore;
4069  } else {
4070  LastLoadStoreInRegion = CurrentLoadStore;
4071  }
4072 }
4073 
4074 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
4075  bool InsertInReadyList,
4076  BoUpSLP *SLP) {
4077  assert(SD->isSchedulingEntity());
4078 
4080  WorkList.push_back(SD);
4081 
4082  while (!WorkList.empty()) {
4083  ScheduleData *SD = WorkList.back();
4084  WorkList.pop_back();
4085 
4086  ScheduleData *BundleMember = SD;
4087  while (BundleMember) {
4088  assert(isInSchedulingRegion(BundleMember));
4089  if (!BundleMember->hasValidDependencies()) {
4090 
4091  DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n");
4092  BundleMember->Dependencies = 0;
4093  BundleMember->resetUnscheduledDeps();
4094 
4095  // Handle def-use chain dependencies.
4096  if (BundleMember->OpValue != BundleMember->Inst) {
4097  ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
4098  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4099  BundleMember->Dependencies++;
4100  ScheduleData *DestBundle = UseSD->FirstInBundle;
4101  if (!DestBundle->IsScheduled)
4102  BundleMember->incrementUnscheduledDeps(1);
4103  if (!DestBundle->hasValidDependencies())
4104  WorkList.push_back(DestBundle);
4105  }
4106  } else {
4107  for (User *U : BundleMember->Inst->users()) {
4108  if (isa<Instruction>(U)) {
4109  ScheduleData *UseSD = getScheduleData(U);
4110  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4111  BundleMember->Dependencies++;
4112  ScheduleData *DestBundle = UseSD->FirstInBundle;
4113  if (!DestBundle->IsScheduled)
4114  BundleMember->incrementUnscheduledDeps(1);
4115  if (!DestBundle->hasValidDependencies())
4116  WorkList.push_back(DestBundle);
4117  }
4118  } else {
4119  // I'm not sure if this can ever happen. But we need to be safe.
4120  // This lets the instruction/bundle never be scheduled and
4121  // eventually disable vectorization.
4122  BundleMember->Dependencies++;
4123  BundleMember->incrementUnscheduledDeps(1);
4124  }
4125  }
4126  }
4127 
4128  // Handle the memory dependencies.
4129  ScheduleData *DepDest = BundleMember->NextLoadStore;
4130  if (DepDest) {
4131  Instruction *SrcInst = BundleMember->Inst;
4132  MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
4133  bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
4134  unsigned numAliased = 0;
4135  unsigned DistToSrc = 1;
4136 
4137  while (DepDest) {
4138  assert(isInSchedulingRegion(DepDest));
4139 
4140  // We have two limits to reduce the complexity:
4141  // 1) AliasedCheckLimit: It's a small limit to reduce calls to
4142  // SLP->isAliased (which is the expensive part in this loop).
4143  // 2) MaxMemDepDistance: It's for very large blocks and it aborts
4144  // the whole loop (even if the loop is fast, it's quadratic).
4145  // It's important for the loop break condition (see below) to
4146  // check this limit even between two read-only instructions.
4147  if (DistToSrc >= MaxMemDepDistance ||
4148  ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
4149  (numAliased >= AliasedCheckLimit ||
4150  SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
4151 
4152  // We increment the counter only if the locations are aliased
4153  // (instead of counting all alias checks). This gives a better
4154  // balance between reduced runtime and accurate dependencies.
4155  numAliased++;
4156 
4157  DepDest->MemoryDependencies.push_back(BundleMember);
4158  BundleMember->Dependencies++;
4159  ScheduleData *DestBundle = DepDest->FirstInBundle;
4160  if (!DestBundle->IsScheduled) {
4161  BundleMember->incrementUnscheduledDeps(1);
4162  }
4163  if (!DestBundle->hasValidDependencies()) {
4164  WorkList.push_back(DestBundle);
4165  }
4166  }
4167  DepDest = DepDest->NextLoadStore;
4168 
4169  // Example, explaining the loop break condition: Let's assume our
4170  // starting instruction is i0 and MaxMemDepDistance = 3.
4171  //
4172  // +--------v--v--v
4173  // i0,i1,i2,i3,i4,i5,i6,i7,i8
4174  // +--------^--^--^
4175  //
4176  // MaxMemDepDistance let us stop alias-checking at i3 and we add
4177  // dependencies from i0 to i3,i4,.. (even if they are not aliased).
4178  // Previously we already added dependencies from i3 to i6,i7,i8
4179  // (because of MaxMemDepDistance). As we added a dependency from
4180  // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4181  // and we can abort this loop at i6.
4182  if (DistToSrc >= 2 * MaxMemDepDistance)
4183  break;
4184  DistToSrc++;
4185  }
4186  }
4187  }
4188  BundleMember = BundleMember->NextInBundle;
4189  }
4190  if (InsertInReadyList && SD->isReady()) {
4191  ReadyInsts.push_back(SD);
4192  DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n");
4193  }
4194  }
4195 }
4196 
4197 void BoUpSLP::BlockScheduling::resetSchedule() {
4198  assert(ScheduleStart &&
4199  "tried to reset schedule on block which has not been scheduled");
4200  for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4201  doForAllOpcodes(I, [&](ScheduleData *SD) {
4202  assert(isInSchedulingRegion(SD) &&
4203  "ScheduleData not in scheduling region");
4204  SD->IsScheduled = false;
4205  SD->resetUnscheduledDeps();
4206  });
4207  }
4208  ReadyInsts.clear();
4209 }
4210 
4211 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4212  if (!BS->ScheduleStart)
4213  return;
4214 
4215  DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
4216 
4217  BS->resetSchedule();
4218 
4219  // For the real scheduling we use a more sophisticated ready-list: it is
4220  // sorted by the original instruction location. This lets the final schedule
4221  // be as close as possible to the original instruction order.
4222  struct ScheduleDataCompare {
4223  bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4224  return SD2->SchedulingPriority < SD1->SchedulingPriority;
4225  }
4226  };
4227  std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4228 
4229  // Ensure that all dependency data is updated and fill the ready-list with
4230  // initial instructions.
4231  int Idx = 0;
4232  int NumToSchedule = 0;
4233  for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4234  I = I->getNextNode()) {
4235  BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4236  assert(SD->isPartOfBundle() ==
4237  (getTreeEntry(SD->Inst) != nullptr) &&
4238  "scheduler and vectorizer bundle mismatch");
4239  SD->FirstInBundle->SchedulingPriority = Idx++;
4240  if (SD->isSchedulingEntity()) {
4241  BS->calculateDependencies(SD, false, this);
4242  NumToSchedule++;
4243  }
4244  });
4245  }
4246  BS->initialFillReadyList(ReadyInsts);
4247 
4248  Instruction *LastScheduledInst = BS->ScheduleEnd;
4249 
4250  // Do the "real" scheduling.
4251  while (!ReadyInsts.empty()) {
4252  ScheduleData *picked = *ReadyInsts.begin();
4253  ReadyInsts.erase(ReadyInsts.begin());
4254 
4255  // Move the scheduled instruction(s) to their dedicated places, if not
4256  // there yet.
4257  ScheduleData *BundleMember = picked;
4258  while (BundleMember) {
4259  Instruction *pickedInst = BundleMember->Inst;
4260  if (LastScheduledInst->getNextNode() != pickedInst) {
4261  BS->BB->getInstList().remove(pickedInst);
4262  BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4263  pickedInst);
4264  }
4265  LastScheduledInst = pickedInst;
4266  BundleMember = BundleMember->NextInBundle;
4267  }
4268 
4269  BS->schedule(picked, ReadyInsts);
4270  NumToSchedule--;
4271  }
4272  assert(NumToSchedule == 0 && "could not schedule all instructions");
4273 
4274  // Avoid duplicate scheduling of the block.
4275  BS->ScheduleStart = nullptr;
4276 }
4277 
4278 unsigned BoUpSLP::getVectorElementSize(Value *V) {
4279  // If V is a store, just return the width of the stored value without
4280  // traversing the expression tree. This is the common case.
4281  if (auto *Store = dyn_cast<StoreInst>(V))
4282  return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4283 
4284  // If V is not a store, we can traverse the expression tree to find loads
4285  // that feed it. The type of the loaded value may indicate a more suitable
4286  // width than V's type. We want to base the vector element size on the width
4287  // of memory operations where possible.
4290  if (auto *I = dyn_cast<Instruction>(V))
4291  Worklist.push_back(I);
4292 
4293  // Traverse the expression tree in bottom-up order looking for loads. If we
4294  // encounter an instruciton we don't yet handle, we give up.
4295  auto MaxWidth = 0u;
4296  auto FoundUnknownInst = false;
4297  while (!Worklist.empty() && !FoundUnknownInst) {
4298  auto *I = Worklist.pop_back_val();
4299  Visited.insert(I);
4300 
4301  // We should only be looking at scalar instructions here. If the current
4302  // instruction has a vector type, give up.
4303  auto *Ty = I->getType();
4304  if (isa<VectorType>(Ty))
4305  FoundUnknownInst = true;
4306 
4307  // If the current instruction is a load, update MaxWidth to reflect the
4308  // width of the loaded value.
4309  else if (isa<LoadInst>(I))
4310  MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4311 
4312  // Otherwise, we need to visit the operands of the instruction. We only
4313  // handle the interesting cases from buildTree here. If an operand is an
4314  // instruction we haven't yet visited, we add it to the worklist.
4315  else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4316  isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4317  for (Use &U : I->operands())
4318  if (auto *J = dyn_cast<Instruction>(U.get()))
4319  if (!Visited.count(J))
4320  Worklist.push_back(J);
4321  }
4322 
4323  // If we don't yet handle the instruction, give up.
4324  else
4325  FoundUnknownInst = true;
4326  }
4327 
4328  // If we didn't encounter a memory access in the expression tree, or if we
4329  // gave up for some reason, just return the width of V.
4330  if (!MaxWidth || FoundUnknownInst)
4331  return DL->getTypeSizeInBits(V->getType());
4332 
4333  // Otherwise, return the maximum width we found.
4334  return MaxWidth;
4335 }
4336 
4337 // Determine if a value V in a vectorizable expression Expr can be demoted to a
4338 // smaller type with a truncation. We collect the values that will be demoted
4339 // in ToDemote and additional roots that require investigating in Roots.
4341  SmallVectorImpl<Value *> &ToDemote,
4342  SmallVectorImpl<Value *> &Roots) {
4343  // We can always demote constants.
4344  if (isa<Constant>(V)) {
4345  ToDemote.push_back(V);
4346  return true;
4347  }
4348 
4349  // If the value is not an instruction in the expression with only one use, it
4350  // cannot be demoted.
4351  auto *I = dyn_cast<Instruction>(V);
4352  if (!I || !I->hasOneUse() || !Expr.count(I))
4353  return false;
4354 
4355  switch (I->getOpcode()) {
4356 
4357  // We can always demote truncations and extensions. Since truncations can
4358  // seed additional demotion, we save the truncated value.
4359  case Instruction::Trunc:
4360  Roots.push_back(I->getOperand(0));
4361  break;
4362  case Instruction::ZExt:
4363  case Instruction::SExt:
4364  break;
4365 
4366  // We can demote certain binary operations if we can demote both of their
4367  // operands.
4368  case Instruction::Add:
4369  case Instruction::Sub:
4370  case Instruction::Mul:
4371  case Instruction::And:
4372  case Instruction::Or:
4373  case Instruction::Xor:
4374  if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4375  !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4376  return false;
4377  break;
4378 
4379  // We can demote selects if we can demote their true and false values.
4380  case Instruction::Select: {
4381  SelectInst *SI = cast<SelectInst>(I);
4382  if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4383  !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4384  return false;
4385  break;
4386  }
4387 
4388  // We can demote phis if we can demote all their incoming operands. Note that
4389  // we don't need to worry about cycles since we ensure single use above.
4390  case Instruction::PHI: {
4391  PHINode *PN = cast<PHINode>(I);
4392  for (Value *IncValue : PN->incoming_values())
4393  if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4394  return false;
4395  break;
4396  }
4397 
4398  // Otherwise, conservatively give up.
4399  default:
4400  return false;
4401  }
4402 
4403  // Record the value that we can demote.
4404  ToDemote.push_back(V);
4405  return true;
4406 }
4407 
4409  // If there are no external uses, the expression tree must be rooted by a
4410  // store. We can't demote in-memory values, so there is nothing to do here.
4411  if (ExternalUses.empty())
4412  return;
4413 
4414  // We only attempt to truncate integer expressions.
4415  auto &TreeRoot = VectorizableTree[0].Scalars;
4416  auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4417  if (!TreeRootIT)
4418  return;
4419 
4420  // If the expression is not rooted by a store, these roots should have
4421  // external uses. We will rely on InstCombine to rewrite the expression in
4422  // the narrower type. However, InstCombine only rewrites single-use values.
4423  // This means that if a tree entry other than a root is used externally, it
4424  // must have multiple uses and InstCombine will not rewrite it. The code
4425  // below ensures that only the roots are used externally.
4426  SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4427  for (auto &EU : ExternalUses)
4428  if (!Expr.erase(EU.Scalar))
4429  return;
4430  if (!Expr.empty())
4431  return;
4432 
4433  // Collect the scalar values of the vectorizable expression. We will use this
4434  // context to determine which values can be demoted. If we see a truncation,
4435  // we mark it as seeding another demotion.
4436  for (auto &Entry : VectorizableTree)
4437  Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4438 
4439  // Ensure the roots of the vectorizable tree don't form a cycle. They must
4440  // have a single external user that is not in the vectorizable tree.
4441  for (auto *Root : TreeRoot)
4442  if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4443  return;
4444 
4445  // Conservatively determine if we can actually truncate the roots of the
4446  // expression. Collect the values that can be demoted in ToDemote and
4447  // additional roots that require investigating in Roots.
4448  SmallVector<Value *, 32> ToDemote;
4450  for (auto *Root : TreeRoot)
4451  if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4452  return;
4453 
4454  // The maximum bit width required to represent all the values that can be
4455  // demoted without loss of precision. It would be safe to truncate the roots
4456  // of the expression to this width.
4457  auto MaxBitWidth = 8u;
4458 
4459  // We first check if all the bits of the roots are demanded. If they're not,
4460  // we can truncate the roots to this narrower type.
4461  for (auto *Root : TreeRoot) {
4462  auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4463  MaxBitWidth = std::max<unsigned>(
4464  Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4465  }
4466 
4467  // True if the roots can be zero-extended back to their original type, rather
4468  // than sign-extended. We know that if the leading bits are not demanded, we
4469  // can safely zero-extend. So we initialize IsKnownPositive to True.
4470  bool IsKnownPositive = true;
4471 
4472  // If all the bits of the roots are demanded, we can try a little harder to
4473  // compute a narrower type. This can happen, for example, if the roots are
4474  // getelementptr indices. InstCombine promotes these indices to the pointer
4475  // width. Thus, all their bits are technically demanded even though the
4476  // address computation might be vectorized in a smaller type.
4477  //
4478  // We start by looking at each entry that can be demoted. We compute the
4479  // maximum bit width required to store the scalar by using ValueTracking to
4480  // compute the number of high-order bits we can truncate.
4481  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
4482  llvm::all_of(TreeRoot, [](Value *R) {
4483  assert(R->hasOneUse() && "Root should have only one use!");
4484  return isa<GetElementPtrInst>(R->user_back());
4485  })) {
4486  MaxBitWidth = 8u;
4487 
4488  // Determine if the sign bit of all the roots is known to be zero. If not,
4489  // IsKnownPositive is set to False.
4490  IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4491  KnownBits Known = computeKnownBits(R, *DL);
4492  return Known.isNonNegative();
4493  });
4494 
4495  // Determine the maximum number of bits required to store the scalar
4496  // values.
4497  for (auto *Scalar : ToDemote) {
4498  auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4499  auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4500  MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4501  }
4502 
4503  // If we can't prove that the sign bit is zero, we must add one to the
4504  // maximum bit width to account for the unknown sign bit. This preserves
4505  // the existing sign bit so we can safely sign-extend the root back to the
4506  // original type. Otherwise, if we know the sign bit is zero, we will
4507  // zero-extend the root instead.
4508  //
4509  // FIXME: This is somewhat suboptimal, as there will be cases where adding
4510  // one to the maximum bit width will yield a larger-than-necessary
4511  // type. In general, we need to add an extra bit only if we can't
4512  // prove that the upper bit of the original type is equal to the
4513  // upper bit of the proposed smaller type. If these two bits are the
4514  // same (either zero or one) we know that sign-extending from the
4515  // smaller type will result in the same value. Here, since we can't
4516  // yet prove this, we are just making the proposed smaller type
4517  // larger to ensure correctness.
4518  if (!IsKnownPositive)
4519  ++MaxBitWidth;
4520  }
4521 
4522  // Round MaxBitWidth up to the next power-of-two.
4523  if (!isPowerOf2_64(MaxBitWidth))
4524  MaxBitWidth = NextPowerOf2(MaxBitWidth);
4525 
4526  // If the maximum bit width we compute is less than the with of the roots'
4527  // type, we can proceed with the narrowing. Otherwise, do nothing.
4528  if (MaxBitWidth >= TreeRootIT->getBitWidth())
4529  return;
4530 
4531  // If we can truncate the root, we must collect additional values that might
4532  // be demoted as a result. That is, those seeded by truncations we will
4533  // modify.
4534  while (!Roots.empty())
4535  collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4536 
4537  // Finally, map the values we can demote to the maximum bit with we computed.
4538  for (auto *Scalar : ToDemote)
4539  MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4540 }
4541 
4542 namespace {
4543 
4544 /// The SLPVectorizer Pass.
4545 struct SLPVectorizer : public FunctionPass {
4546  SLPVectorizerPass Impl;
4547 
4548  /// Pass identification, replacement for typeid
4549  static char ID;
4550 
4551  explicit SLPVectorizer() : FunctionPass(ID) {
4553  }
4554 
4555  bool doInitialization(Module &M) override {
4556  return false;
4557  }
4558 
4559  bool runOnFunction(Function &F) override {
4560  if (skipFunction(F))
4561  return false;
4562 
4563  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4564  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4565  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4566  auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4567  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4568  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4569  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4570  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4571  auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4572  auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4573 
4574  return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4575  }
4576 
4577  void getAnalysisUsage(AnalysisUsage &AU) const override {
4591  AU.setPreservesCFG();
4592  }
4593 };
4594 
4595 } // end anonymous namespace
4596 
4598  auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4599  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4600  auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4601  auto *AA = &AM.getResult<AAManager>(F);
4602  auto *LI = &AM.getResult<LoopAnalysis>(F);
4603  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4604  auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4605  auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4606  auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4607 
4608  bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4609  if (!Changed)
4610  return PreservedAnalyses::all();
4611 
4612  PreservedAnalyses PA;
4613  PA.preserveSet<CFGAnalyses>();
4614  PA.preserve<AAManager>();
4615  PA.preserve<GlobalsAA>();
4616  return PA;
4617 }
4618 
4620  TargetTransformInfo *TTI_,
4621  TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4622  LoopInfo *LI_, DominatorTree *DT_,
4623  AssumptionCache *AC_, DemandedBits *DB_,
4624  OptimizationRemarkEmitter *ORE_) {
4625  SE = SE_;
4626  TTI = TTI_;
4627  TLI = TLI_;
4628  AA = AA_;
4629  LI = LI_;
4630  DT = DT_;
4631  AC = AC_;
4632  DB = DB_;
4633  DL = &F.getParent()->getDataLayout();
4634 
4635  Stores.clear();
4636  GEPs.clear();
4637  bool Changed = false;
4638 
4639  // If the target claims to have no vector registers don't attempt
4640  // vectorization.
4641  if (!TTI->getNumberOfRegisters(true))
4642  return false;
4643 
4644  // Don't vectorize when the attribute NoImplicitFloat is used.
4645  if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4646  return false;
4647 
4648  DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4649 
4650  // Use the bottom up slp vectorizer to construct chains that start with
4651  // store instructions.
4652  BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4653 
4654  // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4655  // delete instructions.
4656 
4657  // Scan the blocks in the function in post order.
4658  for (auto BB : post_order(&F.getEntryBlock())) {
4659  collectSeedInstructions(BB);
4660 
4661  // Vectorize trees that end at stores.
4662  if (!Stores.empty()) {
4663  DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4664  << " underlying objects.\n");
4665  Changed |= vectorizeStoreChains(R);
4666  }
4667 
4668  // Vectorize trees that end at reductions.
4669  Changed |= vectorizeChainsInBlock(BB, R);
4670 
4671  // Vectorize the index computations of getelementptr instructions. This
4672  // is primarily intended to catch gather-like idioms ending at
4673  // non-consecutive loads.
4674  if (!GEPs.empty()) {
4675  DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4676  << " underlying objects.\n");
4677  Changed |= vectorizeGEPIndices(BB, R);
4678  }
4679  }
4680 
4681  if (Changed) {
4682  R.optimizeGatherSequence();
4683  DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4684  DEBUG(verifyFunction(F));
4685  }
4686  return Changed;
4687 }
4688 
4689 /// \brief Check that the Values in the slice in VL array are still existent in
4690 /// the WeakTrackingVH array.
4691 /// Vectorization of part of the VL array may cause later values in the VL array
4692 /// to become invalid. We track when this has happened in the WeakTrackingVH
4693 /// array.
4695  ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4696  unsigned SliceSize) {
4697  VL = VL.slice(SliceBegin, SliceSize);
4698  VH = VH.slice(SliceBegin, SliceSize);
4699  return !std::equal(VL.begin(), VL.end(), VH.begin());
4700 }
4701 
4702 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4703  unsigned VecRegSize) {
4704  const unsigned ChainLen = Chain.size();
4705  DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4706  << "\n");
4707  const unsigned Sz = R.getVectorElementSize(Chain[0]);
4708  const unsigned VF = VecRegSize / Sz;
4709 
4710  if (!isPowerOf2_32(Sz) || VF < 2)
4711  return false;
4712 
4713  // Keep track of values that were deleted by vectorizing in the loop below.
4714  const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4715 
4716  bool Changed = false;
4717  // Look for profitable vectorizable trees at all offsets, starting at zero.
4718  for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4719 
4720  // Check that a previous iteration of this loop did not delete the Value.
4721  if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4722  continue;
4723 
4724  DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4725  << "\n");
4726  ArrayRef<Value *> Operands = Chain.slice(i, VF);
4727 
4728  R.buildTree(Operands);
4729  if (R.isTreeTinyAndNotFullyVectorizable())
4730  continue;
4731 
4732  R.computeMinimumValueSizes();
4733 
4734  int Cost = R.getTreeCost();
4735 
4736  DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
4737  if (Cost < -SLPCostThreshold) {
4738  DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4739 
4740  using namespace ore;
4741 
4742  R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4743  cast<StoreInst>(Chain[i]))
4744  << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4745  << " and with tree size "
4746  << NV("TreeSize", R.getTreeSize()));
4747 
4748  R.vectorizeTree();
4749 
4750  // Move to the next bundle.
4751  i += VF - 1;
4752  Changed = true;
4753  }
4754  }
4755 
4756  return Changed;
4757 }
4758 
4759 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4760  BoUpSLP &R) {
4761  SetVector<StoreInst *> Heads;
4763  SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4764 
4765  // We may run into multiple chains that merge into a single chain. We mark the
4766  // stores that we vectorized so that we don't visit the same store twice.
4767  BoUpSLP::ValueSet VectorizedStores;
4768  bool Changed = false;
4769 
4770  // Do a quadratic search on all of the given stores in reverse order and find
4771  // all of the pairs of stores that follow each other.
4772  SmallVector<unsigned, 16> IndexQueue;
4773  unsigned E = Stores.size();
4774  IndexQueue.resize(E - 1);
4775  for (unsigned I = E; I > 0; --I) {
4776  unsigned Idx = I - 1;
4777  // If a store has multiple consecutive store candidates, search Stores
4778  // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4779  // This is because usually pairing with immediate succeeding or preceding
4780  // candidate create the best chance to find slp vectorization opportunity.
4781  unsigned Offset = 1;
4782  unsigned Cnt = 0;
4783  for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4784  if (Idx >= Offset) {
4785  IndexQueue[Cnt] = Idx - Offset;
4786  ++Cnt;
4787  }
4788  if (Idx + Offset < E) {
4789  IndexQueue[Cnt] = Idx + Offset;
4790  ++Cnt;
4791  }
4792  }
4793 
4794  for (auto K : IndexQueue) {
4795  if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4796  Tails.insert(Stores[Idx]);
4797  Heads.insert(Stores[K]);
4798  ConsecutiveChain[Stores[K]] = Stores[Idx];
4799  break;
4800  }
4801  }
4802  }
4803 
4804  // For stores that start but don't end a link in the chain:
4805  for (auto *SI : llvm::reverse(Heads)) {
4806  if (Tails.count(SI))
4807  continue;
4808 
4809  // We found a store instr that starts a chain. Now follow the chain and try
4810  // to vectorize it.
4811  BoUpSLP::ValueList Operands;
4812  StoreInst *I = SI;
4813  // Collect the chain into a list.
4814  while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4815  Operands.push_back(I);
4816  // Move to the next value in the chain.
4817  I = ConsecutiveChain[I];
4818  }
4819 
4820  // FIXME: Is division-by-2 the correct step? Should we assert that the
4821  // register size is a power-of-2?
4822  for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4823  Size /= 2) {
4824  if (vectorizeStoreChain(Operands, R, Size)) {
4825  // Mark the vectorized stores so that we don't vectorize them again.
4826  VectorizedStores.insert(Operands.begin(), Operands.end());
4827  Changed = true;
4828  break;
4829  }
4830  }
4831  }
4832 
4833  return Changed;
4834 }
4835 
4836 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4837  // Initialize the collections. We will make a single pass over the block.
4838  Stores.clear();
4839  GEPs.clear();
4840 
4841  // Visit the store and getelementptr instructions in BB and organize them in
4842  // Stores and GEPs according to the underlying objects of their pointer
4843  // operands.
4844  for (Instruction &I : *BB) {
4845  // Ignore store instructions that are volatile or have a pointer operand
4846  // that doesn't point to a scalar type.
4847  if (auto *SI = dyn_cast<StoreInst>(&I)) {
4848  if (!SI->isSimple())
4849  continue;
4850  if (!isValidElementType(SI->getValueOperand()->getType()))
4851  continue;
4852  Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4853  }
4854 
4855  // Ignore getelementptr instructions that have more than one index, a
4856  // constant index, or a pointer operand that doesn't point to a scalar
4857  // type.
4858  else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4859  auto Idx = GEP->idx_begin()->get();
4860  if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4861  continue;
4862  if (!isValidElementType(Idx->getType()))
4863  continue;
4864  if (GEP->getType()->isVectorTy())
4865  continue;
4866  GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4867  }
4868  }
4869 }
4870 
4871 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4872  if (!A || !B)
4873  return false;
4874  Value *VL[] = { A, B };
4875  return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4876 }
4877 
4878 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4879  int UserCost, bool AllowReorder) {
4880  if (VL.size() < 2)
4881  return false;
4882 
4883  DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()
4884  << ".\n");
4885 
4886  // Check that all of the parts are scalar instructions of the same type.
4887  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4888  if (!I0)
4889  return false;
4890 
4891  unsigned Opcode0 = I0->getOpcode();
4892 
4893  unsigned Sz = R.getVectorElementSize(I0);
4894  unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4895  unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4896  if (MaxVF < 2) {
4897  R.getORE()->emit([&]() {
4898  return OptimizationRemarkMissed(
4899  SV_NAME, "SmallVF", I0)
4900  << "Cannot SLP vectorize list: vectorization factor "
4901  << "less than 2 is not supported";
4902  });
4903  return false;
4904  }
4905 
4906  for (Value *V : VL) {
4907  Type *Ty = V->getType();
4908  if (!isValidElementType(Ty)) {
4909  // NOTE: the following will give user internal llvm type name, which may not be useful
4910  R.getORE()->emit([&]() {
4911  std::string type_str;
4912  llvm::raw_string_ostream rso(type_str);
4913  Ty->print(rso);
4914  return OptimizationRemarkMissed(
4915  SV_NAME, "UnsupportedType", I0)
4916  << "Cannot SLP vectorize list: type "
4917  << rso.str() + " is unsupported by vectorizer";
4918  });
4919  return false;
4920  }
4921  Instruction *Inst = dyn_cast<Instruction>(V);
4922 
4923  if (!Inst)
4924  return false;
4925  if (Inst->getOpcode() != Opcode0) {
4926  R.getORE()->emit([&]() {
4927  return OptimizationRemarkMissed(
4928  SV_NAME, "InequableTypes", I0)
4929  << "Cannot SLP vectorize list: not all of the "
4930  << "parts of scalar instructions are of the same type: "
4931  << ore::NV("Instruction1Opcode", I0) << " and "
4932  << ore::NV("Instruction2Opcode", Inst);
4933  });
4934  return false;
4935  }
4936  }
4937 
4938  bool Changed = false;
4939  bool CandidateFound = false;
4940  int MinCost = SLPCostThreshold;
4941 
4942  // Keep track of values that were deleted by vectorizing in the loop below.
4943  SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4944 
4945  unsigned NextInst = 0, MaxInst = VL.size();
4946  for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4947  VF /= 2) {
4948  // No actual vectorization should happen, if number of parts is the same as
4949  // provided vectorization factor (i.e. the scalar type is used for vector
4950  // code during codegen).
4951  auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4952  if (TTI->getNumberOfParts(VecTy) == VF)
4953  continue;
4954  for (unsigned I = NextInst; I < MaxInst; ++I) {
4955  unsigned OpsWidth = 0;
4956 
4957  if (I + VF > MaxInst)
4958  OpsWidth = MaxInst - I;
4959  else
4960  OpsWidth = VF;
4961 
4962  if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4963  break;
4964 
4965  // Check that a previous iteration of this loop did not delete the Value.
4966  if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4967  continue;
4968 
4969  DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4970  << "\n");
4971  ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4972 
4973  R.buildTree(Ops);
4974  Optional<ArrayRef<unsigned>> Order = R.bestOrder();
4975  // TODO: check if we can allow reordering for more cases.
4976  if (AllowReorder && Order) {
4977  // TODO: reorder tree nodes without tree rebuilding.
4978  // Conceptually, there is nothing actually preventing us from trying to
4979  // reorder a larger list. In fact, we do exactly this when vectorizing
4980  // reductions. However, at this point, we only expect to get here when
4981  // there are exactly two operations.
4982  assert(Ops.size() == 2);
4983  Value *ReorderedOps[] = {Ops[1], Ops[0]};
4984  R.buildTree(ReorderedOps, None);
4985  }
4986  if (R.isTreeTinyAndNotFullyVectorizable())
4987  continue;
4988 
4989  R.computeMinimumValueSizes();
4990  int Cost = R.getTreeCost() - UserCost;
4991  CandidateFound = true;
4992  MinCost = std::min(MinCost, Cost);
4993 
4994  if (Cost < -SLPCostThreshold) {
4995  DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4996  R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4997  cast<Instruction>(Ops[0]))
4998  << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4999  << " and with tree size "
5000  << ore::NV("TreeSize", R.getTreeSize()));
5001 
5002  R.vectorizeTree();
5003  // Move to the next bundle.
5004  I += VF - 1;
5005  NextInst = I + 1;
5006  Changed = true;
5007  }
5008  }
5009  }
5010 
5011  if (!Changed && CandidateFound) {
5012  R.getORE()->emit([&]() {
5013  return OptimizationRemarkMissed(
5014  SV_NAME, "NotBeneficial", I0)
5015  << "List vectorization was possible but not beneficial with cost "
5016  << ore::NV("Cost", MinCost) << " >= "
5017  << ore::NV("Treshold", -SLPCostThreshold);
5018  });
5019  } else if (!Changed) {
5020  R.getORE()->emit([&]() {
5021  return OptimizationRemarkMissed(
5022  SV_NAME, "NotPossible", I0)
5023  << "Cannot SLP vectorize list: vectorization was impossible"
5024  << " with available vectorization factors";
5025  });
5026  }
5027  return Changed;
5028 }
5029 
5030 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
5031  if (!I)
5032  return false;
5033 
5034  if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
5035  return false;
5036 
5037  Value *P = I->getParent();
5038 
5039  // Vectorize in current basic block only.
5040  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
5041  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
5042  if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
5043  return false;
5044 
5045  // Try to vectorize V.
5046  if (tryToVectorizePair(Op0, Op1, R))
5047  return true;
5048 
5049  auto *A = dyn_cast<BinaryOperator>(Op0);
5050  auto *B = dyn_cast<BinaryOperator>(Op1);
5051  // Try to skip B.
5052  if (B && B->hasOneUse()) {
5053  auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
5054  auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
5055  if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
5056  return true;
5057  if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
5058  return true;
5059  }
5060 
5061  // Try to skip A.
5062  if (A && A->hasOneUse()) {
5063  auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
5064  auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
5065  if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
5066  return true;
5067  if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
5068  return true;
5069  }
5070  return false;
5071 }
5072 
5073 /// \brief Generate a shuffle mask to be used in a reduction tree.
5074 ///
5075 /// \param VecLen The length of the vector to be reduced.
5076 /// \param NumEltsToRdx The number of elements that should be reduced in the
5077 /// vector.
5078 /// \param IsPairwise Whether the reduction is a pairwise or splitting
5079 /// reduction. A pairwise reduction will generate a mask of
5080 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
5081 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
5082 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
5083 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
5084  bool IsPairwise, bool IsLeft,
5085  IRBuilder<> &Builder) {
5086  assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
5087 
5088  SmallVector<Constant *, 32> ShuffleMask(
5089  VecLen, UndefValue::get(Builder.getInt32Ty()));
5090 
5091  if (IsPairwise)
5092  // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
5093  for (unsigned i = 0; i != NumEltsToRdx; ++i)
5094  ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
5095  else
5096  // Move the upper half of the vector to the lower half.
5097  for (unsigned i = 0; i != NumEltsToRdx; ++i)
5098  ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
5099 
5100  return ConstantVector::get(ShuffleMask);
5101 }
5102 
5103 namespace {
5104 
5105 /// Model horizontal reductions.
5106 ///
5107 /// A horizontal reduction is a tree of reduction operations (currently add and
5108 /// fadd) that has operations that can be put into a vector as its leaf.
5109 /// For example, this tree:
5110 ///
5111 /// mul mul mul mul
5112 /// \ / \ /
5113 /// + +
5114 /// \ /
5115 /// +
5116 /// This tree has "mul" as its reduced values and "+" as its reduction
5117 /// operations. A reduction might be feeding into a store or a binary operation
5118 /// feeding a phi.
5119 /// ...
5120 /// \ /
5121 /// +
5122 /// |
5123 /// phi +=
5124 ///
5125 /// Or:
5126 /// ...
5127 /// \ /
5128 /// +
5129 /// |
5130 /// *p =
5131 ///
5132 class HorizontalReduction {
5133  using ReductionOpsType = SmallVector<Value *, 16>;
5134  using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
5135  ReductionOpsListType ReductionOps;
5136  SmallVector<Value *, 32> ReducedVals;
5137  // Use map vector to make stable output.
5139 
5140  /// Kind of the reduction data.
5141  enum ReductionKind {
5142  RK_None, /// Not a reduction.
5143  RK_Arithmetic, /// Binary reduction data.
5144  RK_Min, /// Minimum reduction data.
5145  RK_UMin, /// Unsigned minimum reduction data.
5146  RK_Max, /// Maximum reduction data.
5147  RK_UMax, /// Unsigned maximum reduction data.
5148  };
5149 
5150  /// Contains info about operation, like its opcode, left and right operands.
5151  class OperationData {
5152  /// Opcode of the instruction.
5153  unsigned Opcode = 0;
5154 
5155  /// Left operand of the reduction operation.
5156  Value *LHS = nullptr;
5157 
5158  /// Right operand of the reduction operation.
5159  Value *RHS = nullptr;
5160 
5161  /// Kind of the reduction operation.
5162  ReductionKind Kind = RK_None;
5163 
5164  /// True if float point min/max reduction has no NaNs.
5165  bool NoNaN = false;
5166 
5167  /// Checks if the reduction operation can be vectorized.
5168  bool isVectorizable() const {
5169  return LHS && RHS &&
5170  // We currently only support adds && min/max reductions.
5171  ((Kind == RK_Arithmetic &&
5172  (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
5173  ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5174  (Kind == RK_Min || Kind == RK_Max)) ||
5175  (Opcode == Instruction::ICmp &&
5176  (Kind == RK_UMin || Kind == RK_UMax)));
5177  }
5178 
5179  /// Creates reduction operation with the current opcode.
5180  Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5181  assert(isVectorizable() &&
5182  "Expected add|fadd or min/max reduction operation.");
5183  Value *Cmp;
5184  switch (Kind) {
5185  case RK_Arithmetic:
5186  return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5187  Name);
5188  case RK_Min:
5189  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5190  : Builder.CreateFCmpOLT(LHS, RHS);
5191  break;
5192  case RK_Max:
5193  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5194  : Builder.CreateFCmpOGT(LHS, RHS);
5195  break;
5196  case RK_UMin:
5197  assert(Opcode == Instruction::ICmp && "Expected integer types.");
5198  Cmp = Builder.CreateICmpULT(LHS, RHS);
5199  break;
5200  case RK_UMax:
5201  assert(Opcode == Instruction::ICmp && "Expected integer types.");
5202  Cmp = Builder.CreateICmpUGT(LHS, RHS);
5203  break;
5204  case RK_None:
5205  llvm_unreachable("Unknown reduction operation.");
5206  }
5207  return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5208  }
5209 
5210  public:
5211  explicit OperationData() = default;
5212 
5213  /// Construction for reduced values. They are identified by opcode only and
5214  /// don't have associated LHS/RHS values.
5215  explicit OperationData(Value *V) {
5216  if (auto *I = dyn_cast<Instruction>(V))
5217  Opcode = I->getOpcode();
5218  }
5219 
5220  /// Constructor for reduction operations with opcode and its left and
5221  /// right operands.
5222  OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5223  bool NoNaN = false)
5224  : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5225  assert(Kind != RK_None && "One of the reduction operations is expected.");
5226  }
5227 
5228  explicit operator bool() const { return Opcode; }
5229 
5230  /// Get the index of the first operand.
5231  unsigned getFirstOperandIndex() const {
5232  assert(!!*this && "The opcode is not set.");
5233  switch (Kind) {
5234  case RK_Min:
5235  case RK_UMin:
5236  case RK_Max:
5237  case RK_UMax:
5238  return 1;
5239  case RK_Arithmetic:
5240  case RK_None:
5241  break;
5242  }
5243  return 0;
5244  }
5245 
5246  /// Total number of operands in the reduction operation.
5247  unsigned getNumberOfOperands() const {
5248  assert(Kind != RK_None && !!*this && LHS && RHS &&
5249  "Expected reduction operation.");
5250  switch (Kind) {
5251  case RK_Arithmetic:
5252  return 2;
5253  case RK_Min:
5254  case RK_UMin:
5255  case RK_Max:
5256  case RK_UMax:
5257  return 3;
5258  case RK_None:
5259  break;
5260  }
5261  llvm_unreachable("Reduction kind is not set");
5262  }
5263 
5264  /// Checks if the operation has the same parent as \p P.
5265  bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5266  assert(Kind != RK_None && !!*this && LHS && RHS &&
5267  "Expected reduction operation.");
5268  if (!IsRedOp)
5269  return I->getParent() == P;
5270  switch (Kind) {
5271  case RK_Arithmetic:
5272  // Arithmetic reduction operation must be used once only.
5273  return I->getParent() == P;
5274  case RK_Min:
5275  case RK_UMin:
5276  case RK_Max:
5277  case RK_UMax: {
5278  // SelectInst must be used twice while the condition op must have single
5279  // use only.
5280  auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5281  return I->getParent() == P && Cmp && Cmp->getParent() == P;
5282  }
5283  case RK_None:
5284  break;
5285  }
5286  llvm_unreachable("Reduction kind is not set");
5287  }
5288  /// Expected number of uses for reduction operations/reduced values.
5289  bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5290  assert(Kind != RK_None && !!*this && LHS && RHS &&
5291  "Expected reduction operation.");
5292  switch (Kind) {
5293  case RK_Arithmetic:
5294  return I->hasOneUse();
5295  case RK_Min:
5296  case RK_UMin:
5297  case RK_Max:
5298  case RK_UMax:
5299  return I->hasNUses(2) &&
5300  (!IsReductionOp ||
5301  cast<SelectInst>(I)->getCondition()->hasOneUse());
5302  case RK_None:
5303  break;
5304  }
5305  llvm_unreachable("Reduction kind is not set");
5306  }
5307 
5308  /// Initializes the list of reduction operations.
5309  void initReductionOps(ReductionOpsListType &ReductionOps) {
5310  assert(Kind != RK_None && !!*this && LHS && RHS &&
5311  "Expected reduction operation.");
5312  switch (Kind) {
5313  case RK_Arithmetic:
5314  ReductionOps.assign(1, ReductionOpsType());
5315  break;
5316  case RK_Min:
5317  case RK_UMin:
5318  case RK_Max:
5319  case RK_UMax:
5320  ReductionOps.assign(2, ReductionOpsType());
5321  break;
5322  case RK_None:
5323  llvm_unreachable("Reduction kind is not set");
5324  }
5325  }
5326  /// Add all reduction operations for the reduction instruction \p I.
5327  void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5328  assert(Kind != RK_None && !!*this && LHS && RHS &&
5329  "Expected reduction operation.");
5330  switch (Kind) {
5331  case RK_Arithmetic:
5332  ReductionOps[0].emplace_back(I);
5333  break;
5334  case RK_Min:
5335  case RK_UMin:
5336  case RK_Max:
5337  case RK_UMax:
5338  ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5339  ReductionOps[1].emplace_back(I);
5340  break;
5341  case RK_None:
5342  llvm_unreachable("Reduction kind is not set");
5343  }
5344  }
5345 
5346  /// Checks if instruction is associative and can be vectorized.
5347  bool isAssociative(Instruction *I) const {
5348  assert(Kind != RK_None && *this && LHS && RHS &&
5349  "Expected reduction operation.");
5350  switch (Kind) {
5351  case RK_Arithmetic:
5352  return I->isAssociative();
5353  case RK_Min:
5354  case RK_Max:
5355  return Opcode == Instruction::ICmp ||
5356  cast<Instruction>(I->getOperand(0))->isFast();
5357  case RK_UMin:
5358  case RK_UMax:
5359  assert(Opcode == Instruction::ICmp &&
5360  "Only integer compare operation is expected.");
5361  return true;
5362  case RK_None:
5363  break;
5364  }
5365  llvm_unreachable("Reduction kind is not set");
5366  }
5367 
5368  /// Checks if the reduction operation can be vectorized.
5369  bool isVectorizable(Instruction *I) const {
5370  return isVectorizable() && isAssociative(I);
5371  }
5372 
5373  /// Checks if two operation data are both a reduction op or both a reduced
5374  /// value.
5375  bool operator==(const OperationData &OD) {
5376  assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
5377  "One of the comparing operations is incorrect.");
5378  return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5379  }
5380  bool operator!=(const OperationData &OD) { return !(*this == OD); }
5381  void clear() {
5382  Opcode = 0;
5383  LHS = nullptr;
5384  RHS = nullptr;
5385  Kind = RK_None;
5386  NoNaN = false;
5387  }
5388 
5389  /// Get the opcode of the reduction operation.
5390  unsigned getOpcode() const {
5391  assert(isVectorizable() && "Expected vectorizable operation.");
5392  return Opcode;
5393  }
5394 
5395  /// Get kind of reduction data.
5396  ReductionKind getKind() const { return Kind; }
5397  Value *getLHS() const { return LHS; }
5398  Value *getRHS() const { return RHS; }
5399  Type *getConditionType() const {
5400  switch (Kind) {
5401  case RK_Arithmetic:
5402  return nullptr;
5403  case RK_Min:
5404  case RK_Max:
5405  case RK_UMin:
5406  case RK_UMax:
5407  return CmpInst::makeCmpResultType(LHS->getType());
5408  case RK_None:
5409  break;
5410  }
5411  llvm_unreachable("Reduction kind is not set");
5412  }
5413 
5414  /// Creates reduction operation with the current opcode with the IR flags
5415  /// from \p ReductionOps.
5416  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5417  const ReductionOpsListType &ReductionOps) const {
5418  assert(isVectorizable() &&
5419  "Expected add|fadd or min/max reduction operation.");
5420  auto *Op = createOp(Builder, Name);
5421  switch (Kind) {
5422  case RK_Arithmetic:
5423  propagateIRFlags(Op, ReductionOps[0]);
5424  return Op;
5425  case RK_Min:
5426  case RK_Max:
5427  case RK_UMin:
5428  case RK_UMax:
5429  if (auto *SI = dyn_cast<SelectInst>(Op))
5430  propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5431  propagateIRFlags(Op, ReductionOps[1]);
5432  return Op;
5433  case RK_None:
5434  break;
5435  }
5436  llvm_unreachable("Unknown reduction operation.");
5437  }
5438  /// Creates reduction operation with the current opcode with the IR flags
5439  /// from \p I.
5440  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5441  Instruction *I) const {
5442  assert(isVectorizable() &&
5443  "Expected add|fadd or min/max reduction operation.");
5444  auto *Op = createOp(Builder, Name);
5445  switch (Kind) {
5446  case RK_Arithmetic:
5447  propagateIRFlags(Op, I);
5448  return Op;
5449  case RK_Min:
5450  case RK_Max:
5451  case RK_UMin:
5452  case RK_UMax:
5453  if (auto *SI = dyn_cast<SelectInst>(Op)) {
5454  propagateIRFlags(SI->getCondition(),
5455  cast<SelectInst>(I)->getCondition());
5456  }
5457  propagateIRFlags(Op, I);
5458  return Op;
5459  case RK_None:
5460  break;
5461  }
5462  llvm_unreachable("Unknown reduction operation.");
5463  }
5464 
5465  TargetTransformInfo::ReductionFlags getFlags() const {
5467  Flags.NoNaN = NoNaN;
5468  switch (Kind) {
5469  case RK_Arithmetic:
5470  break;
5471  case RK_Min:
5472  Flags.IsSigned = Opcode == Instruction::ICmp;
5473  Flags.IsMaxOp = false;
5474  break;
5475  case RK_Max:
5476  Flags.IsSigned = Opcode == Instruction::ICmp;
5477  Flags.IsMaxOp = true;
5478  break;
5479  case RK_UMin:
5480  Flags.IsSigned = false;
5481  Flags.IsMaxOp = false;
5482  break;
5483  case RK_UMax:
5484  Flags.IsSigned = false;
5485  Flags.IsMaxOp = true;
5486  break;
5487  case RK_None:
5488  llvm_unreachable("Reduction kind is not set");
5489  }
5490  return Flags;
5491  }
5492  };
5493 
5494  Instruction *ReductionRoot = nullptr;
5495 
5496  /// The operation data of the reduction operation.
5497  OperationData ReductionData;
5498 
5499  /// The operation data of the values we perform a reduction on.
5500  OperationData ReducedValueData;
5501 
5502  /// Should we model this reduction as a pairwise reduction tree or a tree that
5503  /// splits the vector in halves and adds those halves.
5504  bool IsPairwiseReduction = false;
5505 
5506  /// Checks if the ParentStackElem.first should be marked as a reduction
5507  /// operation with an extra argument or as extra argument itself.
5508  void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5509  Value *ExtraArg) {
5510  if (ExtraArgs.count(ParentStackElem.first)) {
5511  ExtraArgs[ParentStackElem.first] = nullptr;
5512  // We ran into something like:
5513  // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5514  // The whole ParentStackElem.first should be considered as an extra value
5515  // in this case.
5516  // Do not perform analysis of remaining operands of ParentStackElem.first
5517  // instruction, this whole instruction is an extra argument.
5518  ParentStackElem.second = ParentStackElem.first->getNumOperands();
5519  } else {
5520  // We ran into something like:
5521  // ParentStackElem.first += ... + ExtraArg + ...
5522  ExtraArgs[ParentStackElem.first] = ExtraArg;
5523  }
5524  }
5525 
5526  static OperationData getOperationData(Value *V) {
5527  if (!V)
5528  return OperationData();
5529 
5530  Value *LHS;
5531  Value *RHS;
5532  if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5533  return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5534  RK_Arithmetic);
5535  }
5536  if (auto *Select = dyn_cast<SelectInst>(V)) {
5537  // Look for a min/max pattern.
5538  if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5539  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5540  } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5541  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5542  } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5543  m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5544  return OperationData(
5545  Instruction::FCmp, LHS, RHS, RK_Min,
5546  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5547  } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5548  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5549  } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5550  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5551  } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5552  m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5553  return OperationData(
5554  Instruction::FCmp, LHS, RHS, RK_Max,
5555  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5556  }
5557  }
5558  return OperationData(V);
5559  }
5560 
5561 public:
5562  HorizontalReduction() = default;
5563 
5564  /// \brief Try to find a reduction tree.
5565  bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5566  assert((!Phi || is_contained(Phi->operands(), B)) &&
5567  "Thi phi needs to use the binary operator");
5568 
5569  ReductionData = getOperationData(B);
5570 
5571  // We could have a initial reductions that is not an add.
5572  // r *= v1 + v2 + v3 + v4
5573  // In such a case start looking for a tree rooted in the first '+'.
5574  if (Phi) {
5575  if (ReductionData.getLHS() == Phi) {
5576  Phi = nullptr;
5577  B = dyn_cast<Instruction>(ReductionData.getRHS());
5578  ReductionData = getOperationData(B);
5579  } else if (ReductionData.getRHS() == Phi) {
5580  Phi = nullptr;
5581  B = dyn_cast<Instruction>(ReductionData.getLHS());
5582  ReductionData = getOperationData(B);
5583  }
5584  }
5585 
5586  if (!ReductionData.isVectorizable(B))
5587  return false;
5588 
5589  Type *Ty = B->getType();
5590  if (!isValidElementType(Ty))
5591  return false;
5592 
5593  ReducedValueData.clear();
5594  ReductionRoot = B;
5595 
5596  // Post order traverse the reduction tree starting at B. We only handle true
5597  // trees containing only binary operators.
5599  Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5600  ReductionData.initReductionOps(ReductionOps);
5601  while (!Stack.empty()) {
5602  Instruction *TreeN = Stack.back().first;
5603  unsigned EdgeToVist = Stack.back().second++;
5604  OperationData OpData = getOperationData(TreeN);
5605  bool IsReducedValue = OpData != ReductionData;
5606 
5607  // Postorder vist.
5608  if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5609  if (IsReducedValue)
5610  ReducedVals.push_back(TreeN);
5611  else {
5612  auto I = ExtraArgs.find(TreeN);
5613  if (I != ExtraArgs.end() && !I->second) {
5614  // Check if TreeN is an extra argument of its parent operation.
5615  if (Stack.size() <= 1) {
5616  // TreeN can't be an extra argument as it is a root reduction
5617  // operation.
5618  return false;
5619  }
5620  // Yes, TreeN is an extra argument, do not add it to a list of
5621  // reduction operations.
5622  // Stack[Stack.size() - 2] always points to the parent operation.
5623  markExtraArg(Stack[Stack.size() - 2], TreeN);
5624  ExtraArgs.erase(TreeN);
5625  } else
5626  ReductionData.addReductionOps(TreeN, ReductionOps);
5627  }
5628  // Retract.
5629  Stack.pop_back();
5630  continue;
5631  }
5632 
5633  // Visit left or right.
5634  Value *NextV = TreeN->getOperand(EdgeToVist);
5635  if (NextV != Phi) {
5636  auto *I = dyn_cast<Instruction>(NextV);
5637  OpData = getOperationData(I);
5638  // Continue analysis if the next operand is a reduction operation or
5639  // (possibly) a reduced value. If the reduced value opcode is not set,
5640  // the first met operation != reduction operation is considered as the
5641  // reduced value class.
5642  if (I && (!ReducedValueData || OpData == ReducedValueData ||
5643  OpData == ReductionData)) {
5644  const bool IsReductionOperation = OpData == ReductionData;
5645  // Only handle trees in the current basic block.
5646  if (!ReductionData.hasSameParent(I, B->getParent(),
5647  IsReductionOperation)) {
5648  // I is an extra argument for TreeN (its parent operation).
5649  markExtraArg(Stack.back(), I);
5650  continue;
5651  }
5652 
5653  // Each tree node needs to have minimal number of users except for the
5654  // ultimate reduction.
5655  if (!ReductionData.hasRequiredNumberOfUses(I,
5656  OpData == ReductionData) &&
5657  I != B) {
5658  // I is an extra argument for TreeN (its parent operation).
5659  markExtraArg(Stack.back(), I);
5660  continue;
5661  }
5662 
5663  if (IsReductionOperation) {
5664  // We need to be able to reassociate the reduction operations.
5665  if (!OpData.isAssociative(I)) {
5666  // I is an extra argument for TreeN (its parent operation).
5667  markExtraArg(Stack.back(), I);
5668  continue;
5669  }
5670  } else if (ReducedValueData &&
5671  ReducedValueData != OpData) {
5672  // Make sure that the opcodes of the operations that we are going to
5673  // reduce match.
5674  // I is an extra argument for TreeN (its parent operation).
5675  markExtraArg(Stack.back(), I);
5676  continue;
5677  } else if (!ReducedValueData)
5678  ReducedValueData = OpData;
5679 
5680  Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5681  continue;
5682  }
5683  }
5684  // NextV is an extra argument for TreeN (its parent operation).
5685  markExtraArg(Stack.back(), NextV);
5686  }
5687  return true;
5688  }
5689 
5690  /// \brief Attempt to vectorize the tree found by
5691  /// matchAssociativeReduction.
5692  bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5693  if (ReducedVals.empty())
5694  return false;
5695 
5696  // If there is a sufficient number of reduction values, reduce
5697  // to a nearby power-of-2. Can safely generate oversized
5698  // vectors and rely on the backend to split them to legal sizes.
5699  unsigned NumReducedVals = ReducedVals.size();
5700  if (NumReducedVals < 4)
5701  return false;
5702 
5703  unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5704 
5705  Value *VectorizedTree = nullptr;
5706  IRBuilder<> Builder(ReductionRoot);
5707  FastMathFlags Unsafe;
5708  Unsafe.setFast();
5709  Builder.setFastMathFlags(Unsafe);
5710  unsigned i = 0;
5711 
5712  BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5713  // The same extra argument may be used several time, so log each attempt
5714  // to use it.
5715  for (auto &Pair : ExtraArgs)
5716  ExternallyUsedValues[Pair.second].push_back(Pair.first);
5717  SmallVector<Value *, 16> IgnoreList;
5718  for (auto &V : ReductionOps)
5719  IgnoreList.append(V.begin(), V.end());
5720  while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5721  auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5722  V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5723  Optional<ArrayRef<unsigned>> Order = V.bestOrder();
5724  // TODO: Handle orders of size less than number of elements in the vector.
5725  if (Order && Order->size() == VL.size()) {
5726  // TODO: reorder tree nodes without tree rebuilding.
5727  SmallVector<Value *, 4> ReorderedOps(VL.size());
5728  llvm::transform(*Order, ReorderedOps.begin(),
5729  [VL](const unsigned Idx) { return VL[Idx]; });
5730  V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList);
5731  }
5732  if (V.isTreeTinyAndNotFullyVectorizable())
5733  break;
5734 
5735  V.computeMinimumValueSizes();
5736 
5737  // Estimate cost.
5738  int Cost =
5739  V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5740  if (Cost >= -SLPCostThreshold) {
5741  V.getORE()->emit([&]() {
5742  return OptimizationRemarkMissed(
5743  SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5744  << "Vectorizing horizontal reduction is possible"
5745  << "but not beneficial with cost "
5746  << ore::NV("Cost", Cost) << " and threshold "
5747  << ore::NV("Threshold", -SLPCostThreshold);
5748  });
5749  break;
5750  }
5751 
5752  DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Cost
5753  << ". (HorRdx)\n");
5754  V.getORE()->emit([&]() {
5755  return OptimizationRemark(
5756  SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5757  << "Vectorized horizontal reduction with cost "
5758  << ore::NV("Cost", Cost) << " and with tree size "
5759  << ore::NV("TreeSize", V.getTreeSize());
5760  });
5761 
5762  // Vectorize a tree.
5763  DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5764  Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5765 
5766  // Emit a reduction.
5767  Value *ReducedSubTree =
5768  emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5769  if (VectorizedTree) {
5770  Builder.SetCurrentDebugLocation(Loc);
5771  OperationData VectReductionData(ReductionData.getOpcode(),
5772  VectorizedTree, ReducedSubTree,
5773  ReductionData.getKind());
5774  VectorizedTree =
5775  VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5776  } else
5777  VectorizedTree = ReducedSubTree;
5778  i += ReduxWidth;
5779  ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5780  }
5781 
5782  if (VectorizedTree) {
5783  // Finish the reduction.
5784  for (; i < NumReducedVals; ++i) {
5785  auto *I = cast<Instruction>(ReducedVals[i]);
5786  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5787  OperationData VectReductionData(ReductionData.getOpcode(),
5788  VectorizedTree, I,
5789  ReductionData.getKind());
5790  VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5791  }
5792  for (auto &Pair : ExternallyUsedValues) {
5793  assert(!Pair.second.empty() &&
5794  "At least one DebugLoc must be inserted");
5795  // Add each externally used value to the final reduction.
5796  for (auto *I : Pair.second) {
5797  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5798  OperationData VectReductionData(ReductionData.getOpcode(),
5799  VectorizedTree, Pair.first,
5800  ReductionData.getKind());
5801  VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5802  }
5803  }
5804  // Update users.
5805  ReductionRoot->replaceAllUsesWith(VectorizedTree);
5806  }
5807  return VectorizedTree != nullptr;
5808  }
5809 
5810  unsigned numReductionValues() const {
5811  return ReducedVals.size();
5812  }
5813 
5814 private:
5815  /// \brief Calculate the cost of a reduction.
5816  int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5817  unsigned ReduxWidth) {
5818  Type *ScalarTy = FirstReducedVal->getType();
5819  Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5820 
5821  int PairwiseRdxCost;
5822  int SplittingRdxCost;
5823  switch (ReductionData.getKind()) {
5824  case RK_Arithmetic:
5825  PairwiseRdxCost =
5826  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5827  /*IsPairwiseForm=*/true);
5828  SplittingRdxCost =
5829  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5830  /*IsPairwiseForm=*/false);
5831  break;
5832  case RK_Min:
5833  case RK_Max:
5834  case RK_UMin:
5835  case RK_UMax: {
5836  Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5837  bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5838  ReductionData.getKind() == RK_UMax;
5839  PairwiseRdxCost =
5840  TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5841  /*IsPairwiseForm=*/true, IsUnsigned);
5842  SplittingRdxCost =
5843  TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5844  /*IsPairwiseForm=*/false, IsUnsigned);
5845  break;
5846  }
5847  case RK_None:
5848  llvm_unreachable("Expected arithmetic or min/max reduction operation");
5849  }
5850 
5851  IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5852  int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5853 
5854  int ScalarReduxCost;
5855  switch (ReductionData.getKind()) {
5856  case RK_Arithmetic:
5857  ScalarReduxCost =
5858  TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5859  break;
5860  case RK_Min:
5861  case RK_Max:
5862  case RK_UMin:
5863  case RK_UMax:
5864  ScalarReduxCost =
5865  TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5866  TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5867  CmpInst::makeCmpResultType(ScalarTy));
5868  break;
5869  case RK_None:
5870  llvm_unreachable("Expected arithmetic or min/max reduction operation");
5871  }
5872  ScalarReduxCost *= (ReduxWidth - 1);
5873 
5874  DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost
5875  << " for reduction that starts with " << *FirstReducedVal
5876  << " (It is a "
5877  << (IsPairwiseReduction ? "pairwise" : "splitting")