Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 2853, column 32
Called C++ object pointer is null

Annotated Source Code

1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11// stores that can be put together into vector-stores. Next, it attempts to
12// construct vectorizable tree using the use-def chains. If a profitable tree
13// was found, the SLP vectorizer performs vectorization on the tree.
14//
15// The pass is inspired by the work described in the paper:
16// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/DenseSet.h"
24#include "llvm/ADT/MapVector.h"
25#include "llvm/ADT/None.h"
26#include "llvm/ADT/Optional.h"
27#include "llvm/ADT/PostOrderIterator.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SetVector.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/iterator.h"
35#include "llvm/ADT/iterator_range.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/Analysis/CodeMetrics.h"
38#include "llvm/Analysis/DemandedBits.h"
39#include "llvm/Analysis/GlobalsModRef.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PassManager.h"
69#include "llvm/IR/PatternMatch.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/IR/Verifier.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/DOTGraphTraits.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/ErrorHandling.h"
83#include "llvm/Support/GraphWriter.h"
84#include "llvm/Support/KnownBits.h"
85#include "llvm/Support/MathExtras.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Transforms/Utils/LoopUtils.h"
88#include "llvm/Transforms/Vectorize.h"
89#include <algorithm>
90#include <cassert>
91#include <cstdint>
92#include <iterator>
93#include <memory>
94#include <set>
95#include <string>
96#include <tuple>
97#include <utility>
98#include <vector>
99
100using namespace llvm;
101using namespace llvm::PatternMatch;
102using namespace slpvectorizer;
103
104#define SV_NAME"slp-vectorizer" "slp-vectorizer"
105#define DEBUG_TYPE"SLP" "SLP"
106
107STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, false}
;
108
109static cl::opt<int>
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
112 "number "));
113
114static cl::opt<bool>
115ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
117
118static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120 cl::desc(
121 "Attempt to vectorize horizontal reductions feeding into a store"));
122
123static cl::opt<int>
124MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
126
127/// Limits the size of scheduling regions in a block.
128/// It avoid long compile times for _very_ large blocks where vector
129/// instructions are spread over a wide range.
130/// This limit is way higher than needed by real-world functions.
131static cl::opt<int>
132ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
134
135static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
138
139static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
142
143static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
146
147static cl::opt<bool>
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
150
151// Limit the number of alias checks. The limit is chosen so that
152// it has no negative effect on the llvm benchmarks.
153static const unsigned AliasedCheckLimit = 10;
154
155// Another limit for the alias checks: The maximum distance between load/store
156// instructions where alias checks are done.
157// This limit is useful for very large basic blocks.
158static const unsigned MaxMemDepDistance = 160;
159
160/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161/// regions to be handled.
162static const int MinScheduleRegionSize = 16;
163
164/// \brief Predicate for the element types that the SLP vectorizer supports.
165///
166/// The most important thing to filter here are types which are invalid in LLVM
167/// vectors. We also filter target specific types which have absolutely no
168/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169/// avoids spending time checking the cost model and realizing that they will
170/// be inevitably scalarized.
171static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
174}
175
176/// \returns true if all of the instructions in \p VL are in the same block or
177/// false otherwise.
178static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180 if (!I0)
181 return false;
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
185 if (!I)
186 return false;
187
188 if (BB != I->getParent())
189 return false;
190 }
191 return true;
192}
193
194/// \returns True if all of the values in \p VL are constants.
195static bool allConstant(ArrayRef<Value *> VL) {
196 for (Value *i : VL)
197 if (!isa<Constant>(i))
198 return false;
199 return true;
200}
201
202/// \returns True if all of the values in \p VL are identical.
203static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
205 if (VL[i] != VL[0])
206 return false;
207 return true;
208}
209
210/// Checks if the vector of instructions can be represented as a shuffle, like:
211/// %x0 = extractelement <4 x i8> %x, i32 0
212/// %x3 = extractelement <4 x i8> %x, i32 3
213/// %y1 = extractelement <4 x i8> %y, i32 1
214/// %y2 = extractelement <4 x i8> %y, i32 2
215/// %x0x0 = mul i8 %x0, %x0
216/// %x3x3 = mul i8 %x3, %x3
217/// %y1y1 = mul i8 %y1, %y1
218/// %y2y2 = mul i8 %y2, %y2
219/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223/// ret <4 x i8> %ins4
224/// can be transformed into:
225/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226/// i32 6>
227/// %2 = mul <4 x i8> %1, %1
228/// ret <4 x i8> %2
229/// We convert this initially to something like:
230/// %x0 = extractelement <4 x i8> %x, i32 0
231/// %x3 = extractelement <4 x i8> %x, i32 3
232/// %y1 = extractelement <4 x i8> %y, i32 1
233/// %y2 = extractelement <4 x i8> %y, i32 2
234/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238/// %5 = mul <4 x i8> %4, %4
239/// %6 = extractelement <4 x i8> %5, i32 0
240/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241/// %7 = extractelement <4 x i8> %5, i32 1
242/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243/// %8 = extractelement <4 x i8> %5, i32 2
244/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245/// %9 = extractelement <4 x i8> %5, i32 3
246/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247/// ret <4 x i8> %ins4
248/// InstCombiner transforms this into a shuffle and vector mul
249static Optional<TargetTransformInfo::ShuffleKind>
250isShuffle(ArrayRef<Value *> VL) {
251 auto *EI0 = cast<ExtractElementInst>(VL[0]);
252 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253 Value *Vec1 = nullptr;
254 Value *Vec2 = nullptr;
255 enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256 ShuffleMode CommonShuffleMode = Unknown;
257 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258 auto *EI = cast<ExtractElementInst>(VL[I]);
259 auto *Vec = EI->getVectorOperand();
260 // All vector operands must have the same number of vector elements.
261 if (Vec->getType()->getVectorNumElements() != Size)
262 return None;
263 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264 if (!Idx)
265 return None;
266 // Undefined behavior if Idx is negative or >= Size.
267 if (Idx->getValue().uge(Size))
268 continue;
269 unsigned IntIdx = Idx->getValue().getZExtValue();
270 // We can extractelement from undef vector.
271 if (isa<UndefValue>(Vec))
272 continue;
273 // For correct shuffling we have to have at most 2 different vector operands
274 // in all extractelement instructions.
275 if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276 return None;
277 if (CommonShuffleMode == Permute)
278 continue;
279 // If the extract index is not the same as the operation number, it is a
280 // permutation.
281 if (IntIdx != I) {
282 CommonShuffleMode = Permute;
283 continue;
284 }
285 // Check the shuffle mode for the current operation.
286 if (!Vec1)
287 Vec1 = Vec;
288 else if (Vec != Vec1)
289 Vec2 = Vec;
290 // Example: shufflevector A, B, <0,5,2,7>
291 // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292 // I is even and IntIdx for B == I - FirstAlternate shuffle.
293 // Example: shufflevector A, B, <4,1,6,3>
294 // I is even and IntIdx for A == I - SecondAlternate shuffle.
295 // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296 const bool IIsEven = I & 1;
297 const bool CurrVecIsA = Vec == Vec1;
298 const bool IIsOdd = !IIsEven;
299 const bool CurrVecIsB = !CurrVecIsA;
300 ShuffleMode CurrentShuffleMode =
301 ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302 : SecondAlternate;
303 // Common mode is not set or the same as the shuffle mode of the current
304 // operation - alternate.
305 if (CommonShuffleMode == Unknown)
306 CommonShuffleMode = CurrentShuffleMode;
307 // Common shuffle mode is not the same as the shuffle mode of the current
308 // operation - permutation.
309 if (CommonShuffleMode != CurrentShuffleMode)
310 CommonShuffleMode = Permute;
311 }
312 // If we're not crossing lanes in different vectors, consider it as blending.
313 if ((CommonShuffleMode == FirstAlternate ||
314 CommonShuffleMode == SecondAlternate) &&
315 Vec2)
316 return TargetTransformInfo::SK_Alternate;
317 // If Vec2 was never used, we have a permutation of a single vector, otherwise
318 // we have permutation of 2 vectors.
319 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
320 : TargetTransformInfo::SK_PermuteSingleSrc;
321}
322
323///\returns Opcode that can be clubbed with \p Op to create an alternate
324/// sequence which can later be merged as a ShuffleVector instruction.
325static unsigned getAltOpcode(unsigned Op) {
326 switch (Op) {
327 case Instruction::FAdd:
328 return Instruction::FSub;
329 case Instruction::FSub:
330 return Instruction::FAdd;
331 case Instruction::Add:
332 return Instruction::Sub;
333 case Instruction::Sub:
334 return Instruction::Add;
335 default:
336 return Op;
337 }
338}
339
340static bool isOdd(unsigned Value) {
341 return Value & 1;
342}
343
344static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345 unsigned CheckedOpcode) {
346 return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347}
348
349/// Checks if the \p Opcode can be considered as an operand of a (possibly)
350/// binary operation \p I.
351/// \returns The code of the binary operation of instruction \p I if the
352/// instruction with \p Opcode can be considered as an operand of \p I with the
353/// default value.
354static unsigned tryToRepresentAsInstArg(unsigned Opcode, Instruction *I) {
355 assert(!sameOpcodeOrAlt(Opcode, getAltOpcode(Opcode), I->getOpcode())((!sameOpcodeOrAlt(Opcode, getAltOpcode(Opcode), I->getOpcode
()) && "Invalid Opcode") ? static_cast<void> (0
) : __assert_fail ("!sameOpcodeOrAlt(Opcode, getAltOpcode(Opcode), I->getOpcode()) && \"Invalid Opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 356, __PRETTY_FUNCTION__))
356 && "Invalid Opcode")((!sameOpcodeOrAlt(Opcode, getAltOpcode(Opcode), I->getOpcode
()) && "Invalid Opcode") ? static_cast<void> (0
) : __assert_fail ("!sameOpcodeOrAlt(Opcode, getAltOpcode(Opcode), I->getOpcode()) && \"Invalid Opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 356, __PRETTY_FUNCTION__))
;
357 if (Opcode != Instruction::PHI && isa<BinaryOperator>(I) &&
358 (I->getType()->isIntegerTy() || cast<FPMathOperator>(I)->isFast()))
359 return I->getOpcode();
360 return 0;
361}
362
363/// Chooses the correct key for scheduling data. If \p Op has the same (or
364/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
365/// OpValue.
366static Value *isOneOf(Value *OpValue, Value *Op) {
367 auto *I = dyn_cast<Instruction>(Op);
368 if (!I)
369 return OpValue;
370 auto *OpInst = cast<Instruction>(OpValue);
371 unsigned OpInstOpcode = OpInst->getOpcode();
372 unsigned IOpcode = I->getOpcode();
373 if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
374 return Op;
375 return OpValue;
376}
377
378namespace {
379
380/// Contains data for the instructions going to be vectorized.
381struct RawInstructionsData {
382 /// Main Opcode of the instructions going to be vectorized.
383 unsigned Opcode = 0;
384 /// Position of the first instruction with the \a Opcode.
385 unsigned OpcodePos = 0;
386 /// Need an additional analysis (if at least one of the instruction is not
387 /// same instruction kind as an instruction at OpcodePos position in the
388 /// list).
389 bool NeedAnalysis = false;
390 /// The list of instructions have some instructions with alternate opcodes.
391 bool HasAltOpcodes = false;
392};
393
394} // end anonymous namespace
395
396/// Checks the list of the vectorized instructions \p VL and returns info about
397/// this list.
398static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
399 auto *I0 = dyn_cast<Instruction>(VL[0]);
400 if (!I0)
401 return {};
402 RawInstructionsData Res;
403 unsigned Opcode = I0->getOpcode();
404 unsigned AltOpcode = getAltOpcode(Opcode);
405 unsigned NewOpcodePos = 0;
406 // Walk through the list of the vectorized instructions
407 // in order to check its structure described by RawInstructionsData.
408 for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
409 auto *I = dyn_cast<Instruction>(VL[Cnt]);
410 if (!I)
411 return {};
412 if (sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode())) {
413 if (Opcode != I->getOpcode()) {
414 Res.HasAltOpcodes = true;
415 if (Res.NeedAnalysis && isOdd(NewOpcodePos))
416 std::swap(Opcode, AltOpcode);
417 }
418 continue;
419 }
420 if (unsigned NewOpcode = tryToRepresentAsInstArg(Opcode, I)) {
421 if (!Instruction::isBinaryOp(Opcode) ||
422 !Instruction::isCommutative(Opcode)) {
423 NewOpcodePos = Cnt;
424 Opcode = NewOpcode;
425 AltOpcode = getAltOpcode(Opcode);
426 Res.NeedAnalysis = true;
427 }
428 } else if (tryToRepresentAsInstArg(I->getOpcode(),
429 cast<Instruction>(VL[NewOpcodePos])))
430 Res.NeedAnalysis = true;
431 else
432 return {};
433 }
434 Res.Opcode = Opcode;
435 Res.OpcodePos = NewOpcodePos;
436 return Res;
437}
438
439namespace {
440
441/// Main data required for vectorization of instructions.
442struct InstructionsState {
443 /// The very first instruction in the list with the main opcode.
444 Value *OpValue = nullptr;
445
446 /// The main opcode for the list of instructions.
447 unsigned Opcode = 0;
448
449 /// Some of the instructions in the list have alternate opcodes.
450 bool IsAltShuffle = false;
451
452 InstructionsState() = default;
453 InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
454 : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
455};
456
457} // end anonymous namespace
458
459/// \returns analysis of the Instructions in \p VL described in
460/// InstructionsState, the Opcode that we suppose the whole list
461/// could be vectorized even if its structure is diverse.
462static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
463 auto Res = getMainOpcode(VL);
464 unsigned Opcode = Res.Opcode;
465 if (!Res.NeedAnalysis && !Res.HasAltOpcodes)
466 return InstructionsState(VL[Res.OpcodePos], Opcode, false);
467 auto *OpInst = cast<Instruction>(VL[Res.OpcodePos]);
468 unsigned AltOpcode = getAltOpcode(Opcode);
469 // Examine each element in the list instructions VL to determine
470 // if some operations there could be considered as an alternative
471 // (for example as subtraction relates to addition operation) or
472 // operation could be an operand of a (possibly) binary operation.
473 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
474 auto *I = cast<Instruction>(VL[Cnt]);
475 unsigned InstOpcode = I->getOpcode();
476 if (Res.NeedAnalysis && !sameOpcodeOrAlt(Opcode, AltOpcode, InstOpcode))
477 if (tryToRepresentAsInstArg(InstOpcode, OpInst))
478 InstOpcode = (Res.HasAltOpcodes && isOdd(Cnt)) ? AltOpcode : Opcode;
479 if ((Res.HasAltOpcodes &&
480 InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
481 (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
482 return InstructionsState(OpInst, 0, false);
483 }
484 }
485 return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
486}
487
488/// \returns true if all of the values in \p VL have the same type or false
489/// otherwise.
490static bool allSameType(ArrayRef<Value *> VL) {
491 Type *Ty = VL[0]->getType();
492 for (int i = 1, e = VL.size(); i < e; i++)
493 if (VL[i]->getType() != Ty)
494 return false;
495
496 return true;
497}
498
499/// \returns True if Extract{Value,Element} instruction extracts element Idx.
500static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
501 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 502, __PRETTY_FUNCTION__))
502 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 502, __PRETTY_FUNCTION__))
;
503 if (Opcode == Instruction::ExtractElement) {
504 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
505 return CI && CI->getZExtValue() == Idx;
506 } else {
507 ExtractValueInst *EI = cast<ExtractValueInst>(E);
508 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
509 }
510}
511
512/// \returns True if in-tree use also needs extract. This refers to
513/// possible scalar operand in vectorized instruction.
514static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
515 TargetLibraryInfo *TLI) {
516 unsigned Opcode = UserInst->getOpcode();
517 switch (Opcode) {
518 case Instruction::Load: {
519 LoadInst *LI = cast<LoadInst>(UserInst);
520 return (LI->getPointerOperand() == Scalar);
521 }
522 case Instruction::Store: {
523 StoreInst *SI = cast<StoreInst>(UserInst);
524 return (SI->getPointerOperand() == Scalar);
525 }
526 case Instruction::Call: {
527 CallInst *CI = cast<CallInst>(UserInst);
528 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
529 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
530 return (CI->getArgOperand(1) == Scalar);
531 }
532 LLVM_FALLTHROUGH[[clang::fallthrough]];
533 }
534 default:
535 return false;
536 }
537}
538
539/// \returns the AA location that is being access by the instruction.
540static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
541 if (StoreInst *SI = dyn_cast<StoreInst>(I))
542 return MemoryLocation::get(SI);
543 if (LoadInst *LI = dyn_cast<LoadInst>(I))
544 return MemoryLocation::get(LI);
545 return MemoryLocation();
546}
547
548/// \returns True if the instruction is not a volatile or atomic load/store.
549static bool isSimple(Instruction *I) {
550 if (LoadInst *LI = dyn_cast<LoadInst>(I))
551 return LI->isSimple();
552 if (StoreInst *SI = dyn_cast<StoreInst>(I))
553 return SI->isSimple();
554 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
555 return !MI->isVolatile();
556 return true;
557}
558
559namespace llvm {
560
561namespace slpvectorizer {
562
563/// Bottom Up SLP Vectorizer.
564class BoUpSLP {
565public:
566 using ValueList = SmallVector<Value *, 8>;
567 using InstrList = SmallVector<Instruction *, 16>;
568 using ValueSet = SmallPtrSet<Value *, 16>;
569 using StoreList = SmallVector<StoreInst *, 8>;
570 using ExtraValueToDebugLocsMap =
571 MapVector<Value *, SmallVector<Instruction *, 2>>;
572
573 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
574 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
575 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
576 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
577 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
578 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
579 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
580 // Use the vector register size specified by the target unless overridden
581 // by a command-line option.
582 // TODO: It would be better to limit the vectorization factor based on
583 // data type rather than just register size. For example, x86 AVX has
584 // 256-bit registers, but it does not support integer operations
585 // at that width (that requires AVX2).
586 if (MaxVectorRegSizeOption.getNumOccurrences())
587 MaxVecRegSize = MaxVectorRegSizeOption;
588 else
589 MaxVecRegSize = TTI->getRegisterBitWidth(true);
590
591 if (MinVectorRegSizeOption.getNumOccurrences())
592 MinVecRegSize = MinVectorRegSizeOption;
593 else
594 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
595 }
596
597 /// \brief Vectorize the tree that starts with the elements in \p VL.
598 /// Returns the vectorized root.
599 Value *vectorizeTree();
600
601 /// Vectorize the tree but with the list of externally used values \p
602 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
603 /// generated extractvalue instructions.
604 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
605
606 /// \returns the cost incurred by unwanted spills and fills, caused by
607 /// holding live values over call sites.
608 int getSpillCost();
609
610 /// \returns the vectorization cost of the subtree that starts at \p VL.
611 /// A negative number means that this is profitable.
612 int getTreeCost();
613
614 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
615 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
616 void buildTree(ArrayRef<Value *> Roots,
617 ArrayRef<Value *> UserIgnoreLst = None);
618
619 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
620 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
621 /// into account (anf updating it, if required) list of externally used
622 /// values stored in \p ExternallyUsedValues.
623 void buildTree(ArrayRef<Value *> Roots,
624 ExtraValueToDebugLocsMap &ExternallyUsedValues,
625 ArrayRef<Value *> UserIgnoreLst = None);
626
627 /// Clear the internal data structures that are created by 'buildTree'.
628 void deleteTree() {
629 VectorizableTree.clear();
630 ScalarToTreeEntry.clear();
631 ExtraScalarToTreeEntry.clear();
632 MustGather.clear();
633 ExternalUses.clear();
634 NumLoadsWantToKeepOrder = 0;
635 NumLoadsWantToChangeOrder = 0;
636 for (auto &Iter : BlocksSchedules) {
637 BlockScheduling *BS = Iter.second.get();
638 BS->clear();
639 }
640 MinBWs.clear();
641 }
642
643 unsigned getTreeSize() const { return VectorizableTree.size(); }
644
645 /// \brief Perform LICM and CSE on the newly generated gather sequences.
646 void optimizeGatherSequence();
647
648 /// \returns true if it is beneficial to reverse the vector order.
649 bool shouldReorder() const {
650 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
651 }
652
653 /// \return The vector element size in bits to use when vectorizing the
654 /// expression tree ending at \p V. If V is a store, the size is the width of
655 /// the stored value. Otherwise, the size is the width of the largest loaded
656 /// value reaching V. This method is used by the vectorizer to calculate
657 /// vectorization factors.
658 unsigned getVectorElementSize(Value *V);
659
660 /// Compute the minimum type sizes required to represent the entries in a
661 /// vectorizable tree.
662 void computeMinimumValueSizes();
663
664 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
665 unsigned getMaxVecRegSize() const {
666 return MaxVecRegSize;
667 }
668
669 // \returns minimum vector register size as set by cl::opt.
670 unsigned getMinVecRegSize() const {
671 return MinVecRegSize;
672 }
673
674 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
675 ///
676 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
677 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
678
679 /// \returns True if the VectorizableTree is both tiny and not fully
680 /// vectorizable. We do not vectorize such trees.
681 bool isTreeTinyAndNotFullyVectorizable();
682
683 OptimizationRemarkEmitter *getORE() { return ORE; }
684
685private:
686 struct TreeEntry;
687
688 /// Checks if all users of \p I are the part of the vectorization tree.
689 bool areAllUsersVectorized(Instruction *I) const;
690
691 /// \returns the cost of the vectorizable entry.
692 int getEntryCost(TreeEntry *E);
693
694 /// This is the recursive part of buildTree.
695 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
696
697 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
698 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
699 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const;
700
701 /// Vectorize a single entry in the tree.
702 Value *vectorizeTree(TreeEntry *E);
703
704 /// Vectorize a single entry in the tree, starting in \p VL.
705 Value *vectorizeTree(ArrayRef<Value *> VL);
706
707 /// \returns the pointer to the vectorized value if \p VL is already
708 /// vectorized, or NULL. They may happen in cycles.
709 Value *alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const;
710
711 /// \returns the scalarization cost for this type. Scalarization in this
712 /// context means the creation of vectors from a group of scalars.
713 int getGatherCost(Type *Ty);
714
715 /// \returns the scalarization cost for this list of values. Assuming that
716 /// this subtree gets vectorized, we may need to extract the values from the
717 /// roots. This method calculates the cost of extracting the values.
718 int getGatherCost(ArrayRef<Value *> VL);
719
720 /// \brief Set the Builder insert point to one after the last instruction in
721 /// the bundle
722 void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
723
724 /// \returns a vector from a collection of scalars in \p VL.
725 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
726
727 /// \returns whether the VectorizableTree is fully vectorizable and will
728 /// be beneficial even the tree height is tiny.
729 bool isFullyVectorizableTinyTree();
730
731 /// \reorder commutative operands in alt shuffle if they result in
732 /// vectorized code.
733 void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
734 SmallVectorImpl<Value *> &Left,
735 SmallVectorImpl<Value *> &Right);
736
737 /// \reorder commutative operands to get better probability of
738 /// generating vectorized code.
739 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
740 SmallVectorImpl<Value *> &Left,
741 SmallVectorImpl<Value *> &Right);
742 struct TreeEntry {
743 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
744
745 /// \returns true if the scalars in VL are equal to this entry.
746 bool isSame(ArrayRef<Value *> VL) const {
747 assert(VL.size() == Scalars.size() && "Invalid size")((VL.size() == Scalars.size() && "Invalid size") ? static_cast
<void> (0) : __assert_fail ("VL.size() == Scalars.size() && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 747, __PRETTY_FUNCTION__))
;
748 return std::equal(VL.begin(), VL.end(), Scalars.begin());
749 }
750
751 /// A vector of scalars.
752 ValueList Scalars;
753
754 /// The Scalars are vectorized into this value. It is initialized to Null.
755 Value *VectorizedValue = nullptr;
756
757 /// Do we need to gather this sequence ?
758 bool NeedToGather = false;
759
760 /// Points back to the VectorizableTree.
761 ///
762 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
763 /// to be a pointer and needs to be able to initialize the child iterator.
764 /// Thus we need a reference back to the container to translate the indices
765 /// to entries.
766 std::vector<TreeEntry> &Container;
767
768 /// The TreeEntry index containing the user of this entry. We can actually
769 /// have multiple users so the data structure is not truly a tree.
770 SmallVector<int, 1> UserTreeIndices;
771
772 /// Info about instruction in this tree entry.
773 InstructionsState State;
774 };
775
776 /// Create a new VectorizableTree entry.
777 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
778 int &UserTreeIdx, const InstructionsState &S) {
779 assert((!Vectorized || S.Opcode != 0) &&(((!Vectorized || S.Opcode != 0) && "Vectorized TreeEntry without opcode"
) ? static_cast<void> (0) : __assert_fail ("(!Vectorized || S.Opcode != 0) && \"Vectorized TreeEntry without opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 780, __PRETTY_FUNCTION__))
780 "Vectorized TreeEntry without opcode")(((!Vectorized || S.Opcode != 0) && "Vectorized TreeEntry without opcode"
) ? static_cast<void> (0) : __assert_fail ("(!Vectorized || S.Opcode != 0) && \"Vectorized TreeEntry without opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 780, __PRETTY_FUNCTION__))
;
781 VectorizableTree.emplace_back(VectorizableTree);
782 int idx = VectorizableTree.size() - 1;
783 TreeEntry *Last = &VectorizableTree[idx];
784 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
785 Last->NeedToGather = !Vectorized;
786 if (Vectorized) {
787 Last->State = S;
788 unsigned AltOpcode = getAltOpcode(S.Opcode);
789 for (int i = 0, e = VL.size(); i != e; ++i) {
790 unsigned RealOpcode =
791 (S.IsAltShuffle && isOdd(i)) ? AltOpcode : S.Opcode;
792 Value *Key = (cast<Instruction>(VL[i])->getOpcode() == RealOpcode)
793 ? VL[i]
794 : S.OpValue;
795 assert(!getTreeEntry(VL[i], Key) && "Scalar already in tree!")((!getTreeEntry(VL[i], Key) && "Scalar already in tree!"
) ? static_cast<void> (0) : __assert_fail ("!getTreeEntry(VL[i], Key) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 795, __PRETTY_FUNCTION__))
;
796 if (VL[i] == Key)
797 ScalarToTreeEntry[Key] = idx;
798 else
799 ExtraScalarToTreeEntry[VL[i]][Key] = idx;
800 }
801 } else {
802 Last->State.Opcode = 0;
803 Last->State.OpValue = VL[0];
804 Last->State.IsAltShuffle = false;
805 MustGather.insert(VL.begin(), VL.end());
806 }
807
808 if (UserTreeIdx >= 0)
809 Last->UserTreeIndices.push_back(UserTreeIdx);
810 UserTreeIdx = idx;
811 return Last;
812 }
813
814 /// -- Vectorization State --
815 /// Holds all of the tree entries.
816 std::vector<TreeEntry> VectorizableTree;
817
818 TreeEntry *getTreeEntry(Value *V) {
819 auto I = ScalarToTreeEntry.find(V);
820 if (I != ScalarToTreeEntry.end())
821 return &VectorizableTree[I->second];
822 return nullptr;
823 }
824
825 const TreeEntry *getTreeEntry(Value *V) const {
826 auto I = ScalarToTreeEntry.find(V);
827 if (I != ScalarToTreeEntry.end())
828 return &VectorizableTree[I->second];
829 return nullptr;
830 }
831
832 TreeEntry *getTreeEntry(Value *V, Value *OpValue) {
833 if (V == OpValue)
834 return getTreeEntry(V);
835 auto I = ExtraScalarToTreeEntry.find(V);
836 if (I != ExtraScalarToTreeEntry.end()) {
837 auto &STT = I->second;
838 auto STTI = STT.find(OpValue);
839 if (STTI != STT.end())
840 return &VectorizableTree[STTI->second];
841 }
842 return nullptr;
843 }
844
845 /// Maps a specific scalar to its tree entry.
846 SmallDenseMap<Value *, int> ScalarToTreeEntry;
847
848 /// Maps a specific scalar to its tree entry(s) with leading scalar.
849 SmallDenseMap<Value *, SmallDenseMap<Value *, int>> ExtraScalarToTreeEntry;
850
851 /// A list of scalars that we found that we need to keep as scalars.
852 ValueSet MustGather;
853
854 /// This POD struct describes one external user in the vectorized tree.
855 struct ExternalUser {
856 ExternalUser(Value *S, llvm::User *U, int L)
857 : Scalar(S), User(U), Lane(L) {}
858
859 // Which scalar in our function.
860 Value *Scalar;
861
862 // Which user that uses the scalar.
863 llvm::User *User;
864
865 // Which lane does the scalar belong to.
866 int Lane;
867 };
868 using UserList = SmallVector<ExternalUser, 16>;
869
870 /// Checks if two instructions may access the same memory.
871 ///
872 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
873 /// is invariant in the calling loop.
874 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
875 Instruction *Inst2) {
876 // First check if the result is already in the cache.
877 AliasCacheKey key = std::make_pair(Inst1, Inst2);
878 Optional<bool> &result = AliasCache[key];
879 if (result.hasValue()) {
880 return result.getValue();
881 }
882 MemoryLocation Loc2 = getLocation(Inst2, AA);
883 bool aliased = true;
884 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
885 // Do the alias check.
886 aliased = AA->alias(Loc1, Loc2);
887 }
888 // Store the result in the cache.
889 result = aliased;
890 return aliased;
891 }
892
893 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
894
895 /// Cache for alias results.
896 /// TODO: consider moving this to the AliasAnalysis itself.
897 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
898
899 /// Removes an instruction from its block and eventually deletes it.
900 /// It's like Instruction::eraseFromParent() except that the actual deletion
901 /// is delayed until BoUpSLP is destructed.
902 /// This is required to ensure that there are no incorrect collisions in the
903 /// AliasCache, which can happen if a new instruction is allocated at the
904 /// same address as a previously deleted instruction.
905 void eraseInstruction(Instruction *I) {
906 I->removeFromParent();
907 I->dropAllReferences();
908 DeletedInstructions.emplace_back(I);
909 }
910
911 /// Temporary store for deleted instructions. Instructions will be deleted
912 /// eventually when the BoUpSLP is destructed.
913 SmallVector<unique_value, 8> DeletedInstructions;
914
915 /// A list of values that need to extracted out of the tree.
916 /// This list holds pairs of (Internal Scalar : External User). External User
917 /// can be nullptr, it means that this Internal Scalar will be used later,
918 /// after vectorization.
919 UserList ExternalUses;
920
921 /// Values used only by @llvm.assume calls.
922 SmallPtrSet<const Value *, 32> EphValues;
923
924 /// Holds all of the instructions that we gathered.
925 SetVector<Instruction *> GatherSeq;
926
927 /// A list of blocks that we are going to CSE.
928 SetVector<BasicBlock *> CSEBlocks;
929
930 /// Contains all scheduling relevant data for an instruction.
931 /// A ScheduleData either represents a single instruction or a member of an
932 /// instruction bundle (= a group of instructions which is combined into a
933 /// vector instruction).
934 struct ScheduleData {
935 // The initial value for the dependency counters. It means that the
936 // dependencies are not calculated yet.
937 enum { InvalidDeps = -1 };
938
939 ScheduleData() = default;
940
941 void init(int BlockSchedulingRegionID, Value *OpVal) {
942 FirstInBundle = this;
943 NextInBundle = nullptr;
944 NextLoadStore = nullptr;
945 IsScheduled = false;
946 SchedulingRegionID = BlockSchedulingRegionID;
947 UnscheduledDepsInBundle = UnscheduledDeps;
948 clearDependencies();
949 OpValue = OpVal;
950 }
951
952 /// Returns true if the dependency information has been calculated.
953 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
954
955 /// Returns true for single instructions and for bundle representatives
956 /// (= the head of a bundle).
957 bool isSchedulingEntity() const { return FirstInBundle == this; }
958
959 /// Returns true if it represents an instruction bundle and not only a
960 /// single instruction.
961 bool isPartOfBundle() const {
962 return NextInBundle != nullptr || FirstInBundle != this;
963 }
964
965 /// Returns true if it is ready for scheduling, i.e. it has no more
966 /// unscheduled depending instructions/bundles.
967 bool isReady() const {
968 assert(isSchedulingEntity() &&((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 969, __PRETTY_FUNCTION__))
969 "can't consider non-scheduling entity for ready list")((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 969, __PRETTY_FUNCTION__))
;
970 return UnscheduledDepsInBundle == 0 && !IsScheduled;
971 }
972
973 /// Modifies the number of unscheduled dependencies, also updating it for
974 /// the whole bundle.
975 int incrementUnscheduledDeps(int Incr) {
976 UnscheduledDeps += Incr;
977 return FirstInBundle->UnscheduledDepsInBundle += Incr;
978 }
979
980 /// Sets the number of unscheduled dependencies to the number of
981 /// dependencies.
982 void resetUnscheduledDeps() {
983 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
984 }
985
986 /// Clears all dependency information.
987 void clearDependencies() {
988 Dependencies = InvalidDeps;
989 resetUnscheduledDeps();
990 MemoryDependencies.clear();
991 }
992
993 void dump(raw_ostream &os) const {
994 if (!isSchedulingEntity()) {
995 os << "/ " << *Inst;
996 } else if (NextInBundle) {
997 os << '[' << *Inst;
998 ScheduleData *SD = NextInBundle;
999 while (SD) {
1000 os << ';' << *SD->Inst;
1001 SD = SD->NextInBundle;
1002 }
1003 os << ']';
1004 } else {
1005 os << *Inst;
1006 }
1007 }
1008
1009 Instruction *Inst = nullptr;
1010
1011 /// Points to the head in an instruction bundle (and always to this for
1012 /// single instructions).
1013 ScheduleData *FirstInBundle = nullptr;
1014
1015 /// Single linked list of all instructions in a bundle. Null if it is a
1016 /// single instruction.
1017 ScheduleData *NextInBundle = nullptr;
1018
1019 /// Single linked list of all memory instructions (e.g. load, store, call)
1020 /// in the block - until the end of the scheduling region.
1021 ScheduleData *NextLoadStore = nullptr;
1022
1023 /// The dependent memory instructions.
1024 /// This list is derived on demand in calculateDependencies().
1025 SmallVector<ScheduleData *, 4> MemoryDependencies;
1026
1027 /// This ScheduleData is in the current scheduling region if this matches
1028 /// the current SchedulingRegionID of BlockScheduling.
1029 int SchedulingRegionID = 0;
1030
1031 /// Used for getting a "good" final ordering of instructions.
1032 int SchedulingPriority = 0;
1033
1034 /// The number of dependencies. Constitutes of the number of users of the
1035 /// instruction plus the number of dependent memory instructions (if any).
1036 /// This value is calculated on demand.
1037 /// If InvalidDeps, the number of dependencies is not calculated yet.
1038 int Dependencies = InvalidDeps;
1039
1040 /// The number of dependencies minus the number of dependencies of scheduled
1041 /// instructions. As soon as this is zero, the instruction/bundle gets ready
1042 /// for scheduling.
1043 /// Note that this is negative as long as Dependencies is not calculated.
1044 int UnscheduledDeps = InvalidDeps;
1045
1046 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
1047 /// single instructions.
1048 int UnscheduledDepsInBundle = InvalidDeps;
1049
1050 /// True if this instruction is scheduled (or considered as scheduled in the
1051 /// dry-run).
1052 bool IsScheduled = false;
1053
1054 /// Opcode of the current instruction in the schedule data.
1055 Value *OpValue = nullptr;
1056 };
1057
1058#ifndef NDEBUG
1059 friend inline raw_ostream &operator<<(raw_ostream &os,
1060 const BoUpSLP::ScheduleData &SD) {
1061 SD.dump(os);
1062 return os;
1063 }
1064#endif
1065
1066 friend struct GraphTraits<BoUpSLP *>;
1067 friend struct DOTGraphTraits<BoUpSLP *>;
1068
1069 /// Contains all scheduling data for a basic block.
1070 struct BlockScheduling {
1071 BlockScheduling(BasicBlock *BB)
1072 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
1073
1074 void clear() {
1075 ReadyInsts.clear();
1076 ScheduleStart = nullptr;
1077 ScheduleEnd = nullptr;
1078 FirstLoadStoreInRegion = nullptr;
1079 LastLoadStoreInRegion = nullptr;
1080
1081 // Reduce the maximum schedule region size by the size of the
1082 // previous scheduling run.
1083 ScheduleRegionSizeLimit -= ScheduleRegionSize;
1084 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1085 ScheduleRegionSizeLimit = MinScheduleRegionSize;
1086 ScheduleRegionSize = 0;
1087
1088 // Make a new scheduling region, i.e. all existing ScheduleData is not
1089 // in the new region yet.
1090 ++SchedulingRegionID;
1091 }
1092
1093 ScheduleData *getScheduleData(Value *V) {
1094 ScheduleData *SD = ScheduleDataMap[V];
1095 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1096 return SD;
1097 return nullptr;
1098 }
1099
1100 ScheduleData *getScheduleData(Value *V, Value *Key) {
1101 if (V == Key)
1102 return getScheduleData(V);
1103 auto I = ExtraScheduleDataMap.find(V);
1104 if (I != ExtraScheduleDataMap.end()) {
1105 ScheduleData *SD = I->second[Key];
1106 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1107 return SD;
1108 }
1109 return nullptr;
1110 }
1111
1112 bool isInSchedulingRegion(ScheduleData *SD) {
1113 return SD->SchedulingRegionID == SchedulingRegionID;
1114 }
1115
1116 /// Marks an instruction as scheduled and puts all dependent ready
1117 /// instructions into the ready-list.
1118 template <typename ReadyListType>
1119 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1120 SD->IsScheduled = true;
1121 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
1122
1123 ScheduleData *BundleMember = SD;
1124 while (BundleMember) {
1125 if (BundleMember->Inst != BundleMember->OpValue) {
1126 BundleMember = BundleMember->NextInBundle;
1127 continue;
1128 }
1129 // Handle the def-use chain dependencies.
1130 for (Use &U : BundleMember->Inst->operands()) {
1131 auto *I = dyn_cast<Instruction>(U.get());
1132 if (!I)
1133 continue;
1134 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1135 if (OpDef && OpDef->hasValidDependencies() &&
1136 OpDef->incrementUnscheduledDeps(-1) == 0) {
1137 // There are no more unscheduled dependencies after
1138 // decrementing, so we can put the dependent instruction
1139 // into the ready list.
1140 ScheduleData *DepBundle = OpDef->FirstInBundle;
1141 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1142, __PRETTY_FUNCTION__))
1142 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1142, __PRETTY_FUNCTION__))
;
1143 ReadyList.insert(DepBundle);
1144 DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
1145 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
1146 }
1147 });
1148 }
1149 // Handle the memory dependencies.
1150 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1151 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1152 // There are no more unscheduled dependencies after decrementing,
1153 // so we can put the dependent instruction into the ready list.
1154 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1155 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1156, __PRETTY_FUNCTION__))
1156 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1156, __PRETTY_FUNCTION__))
;
1157 ReadyList.insert(DepBundle);
1158 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundledo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
1159 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
1160 }
1161 }
1162 BundleMember = BundleMember->NextInBundle;
1163 }
1164 }
1165
1166 void doForAllOpcodes(Value *V,
1167 function_ref<void(ScheduleData *SD)> Action) {
1168 if (ScheduleData *SD = getScheduleData(V))
1169 Action(SD);
1170 auto I = ExtraScheduleDataMap.find(V);
1171 if (I != ExtraScheduleDataMap.end())
1172 for (auto &P : I->second)
1173 if (P.second->SchedulingRegionID == SchedulingRegionID)
1174 Action(P.second);
1175 }
1176
1177 /// Put all instructions into the ReadyList which are ready for scheduling.
1178 template <typename ReadyListType>
1179 void initialFillReadyList(ReadyListType &ReadyList) {
1180 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1181 doForAllOpcodes(I, [&](ScheduleData *SD) {
1182 if (SD->isSchedulingEntity() && SD->isReady()) {
1183 ReadyList.insert(SD);
1184 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
1185 }
1186 });
1187 }
1188 }
1189
1190 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1191 /// cyclic dependencies. This is only a dry-run, no instructions are
1192 /// actually moved at this stage.
1193 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1194
1195 /// Un-bundles a group of instructions.
1196 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1197
1198 /// Allocates schedule data chunk.
1199 ScheduleData *allocateScheduleDataChunks();
1200
1201 /// Extends the scheduling region so that V is inside the region.
1202 /// \returns true if the region size is within the limit.
1203 bool extendSchedulingRegion(Value *V, Value *OpValue);
1204
1205 /// Initialize the ScheduleData structures for new instructions in the
1206 /// scheduling region.
1207 void initScheduleData(Instruction *FromI, Instruction *ToI,
1208 ScheduleData *PrevLoadStore,
1209 ScheduleData *NextLoadStore);
1210
1211 /// Updates the dependency information of a bundle and of all instructions/
1212 /// bundles which depend on the original bundle.
1213 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1214 BoUpSLP *SLP);
1215
1216 /// Sets all instruction in the scheduling region to un-scheduled.
1217 void resetSchedule();
1218
1219 BasicBlock *BB;
1220
1221 /// Simple memory allocation for ScheduleData.
1222 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1223
1224 /// The size of a ScheduleData array in ScheduleDataChunks.
1225 int ChunkSize;
1226
1227 /// The allocator position in the current chunk, which is the last entry
1228 /// of ScheduleDataChunks.
1229 int ChunkPos;
1230
1231 /// Attaches ScheduleData to Instruction.
1232 /// Note that the mapping survives during all vectorization iterations, i.e.
1233 /// ScheduleData structures are recycled.
1234 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1235
1236 /// Attaches ScheduleData to Instruction with the leading key.
1237 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1238 ExtraScheduleDataMap;
1239
1240 struct ReadyList : SmallVector<ScheduleData *, 8> {
1241 void insert(ScheduleData *SD) { push_back(SD); }
1242 };
1243
1244 /// The ready-list for scheduling (only used for the dry-run).
1245 ReadyList ReadyInsts;
1246
1247 /// The first instruction of the scheduling region.
1248 Instruction *ScheduleStart = nullptr;
1249
1250 /// The first instruction _after_ the scheduling region.
1251 Instruction *ScheduleEnd = nullptr;
1252
1253 /// The first memory accessing instruction in the scheduling region
1254 /// (can be null).
1255 ScheduleData *FirstLoadStoreInRegion = nullptr;
1256
1257 /// The last memory accessing instruction in the scheduling region
1258 /// (can be null).
1259 ScheduleData *LastLoadStoreInRegion = nullptr;
1260
1261 /// The current size of the scheduling region.
1262 int ScheduleRegionSize = 0;
1263
1264 /// The maximum size allowed for the scheduling region.
1265 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1266
1267 /// The ID of the scheduling region. For a new vectorization iteration this
1268 /// is incremented which "removes" all ScheduleData from the region.
1269 // Make sure that the initial SchedulingRegionID is greater than the
1270 // initial SchedulingRegionID in ScheduleData (which is 0).
1271 int SchedulingRegionID = 1;
1272 };
1273
1274 /// Attaches the BlockScheduling structures to basic blocks.
1275 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1276
1277 /// Performs the "real" scheduling. Done before vectorization is actually
1278 /// performed in a basic block.
1279 void scheduleBlock(BlockScheduling *BS);
1280
1281 /// List of users to ignore during scheduling and that don't need extracting.
1282 ArrayRef<Value *> UserIgnoreList;
1283
1284 // Number of load bundles that contain consecutive loads.
1285 int NumLoadsWantToKeepOrder = 0;
1286
1287 // Number of load bundles that contain consecutive loads in reversed order.
1288 int NumLoadsWantToChangeOrder = 0;
1289
1290 // Analysis and block reference.
1291 Function *F;
1292 ScalarEvolution *SE;
1293 TargetTransformInfo *TTI;
1294 TargetLibraryInfo *TLI;
1295 AliasAnalysis *AA;
1296 LoopInfo *LI;
1297 DominatorTree *DT;
1298 AssumptionCache *AC;
1299 DemandedBits *DB;
1300 const DataLayout *DL;
1301 OptimizationRemarkEmitter *ORE;
1302
1303 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1304 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1305
1306 /// Instruction builder to construct the vectorized tree.
1307 IRBuilder<> Builder;
1308
1309 /// A map of scalar integer values to the smallest bit width with which they
1310 /// can legally be represented. The values map to (width, signed) pairs,
1311 /// where "width" indicates the minimum bit width and "signed" is True if the
1312 /// value must be signed-extended, rather than zero-extended, back to its
1313 /// original width.
1314 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1315};
1316
1317} // end namespace slpvectorizer
1318
1319template <> struct GraphTraits<BoUpSLP *> {
1320 using TreeEntry = BoUpSLP::TreeEntry;
1321
1322 /// NodeRef has to be a pointer per the GraphWriter.
1323 using NodeRef = TreeEntry *;
1324
1325 /// \brief Add the VectorizableTree to the index iterator to be able to return
1326 /// TreeEntry pointers.
1327 struct ChildIteratorType
1328 : public iterator_adaptor_base<ChildIteratorType,
1329 SmallVector<int, 1>::iterator> {
1330 std::vector<TreeEntry> &VectorizableTree;
1331
1332 ChildIteratorType(SmallVector<int, 1>::iterator W,
1333 std::vector<TreeEntry> &VT)
1334 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1335
1336 NodeRef operator*() { return &VectorizableTree[*I]; }
1337 };
1338
1339 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1340
1341 static ChildIteratorType child_begin(NodeRef N) {
1342 return {N->UserTreeIndices.begin(), N->Container};
1343 }
1344
1345 static ChildIteratorType child_end(NodeRef N) {
1346 return {N->UserTreeIndices.end(), N->Container};
1347 }
1348
1349 /// For the node iterator we just need to turn the TreeEntry iterator into a
1350 /// TreeEntry* iterator so that it dereferences to NodeRef.
1351 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1352
1353 static nodes_iterator nodes_begin(BoUpSLP *R) {
1354 return nodes_iterator(R->VectorizableTree.begin());
1355 }
1356
1357 static nodes_iterator nodes_end(BoUpSLP *R) {
1358 return nodes_iterator(R->VectorizableTree.end());
1359 }
1360
1361 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1362};
1363
1364template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1365 using TreeEntry = BoUpSLP::TreeEntry;
1366
1367 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1368
1369 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1370 std::string Str;
1371 raw_string_ostream OS(Str);
1372 if (isSplat(Entry->Scalars)) {
1373 OS << "<splat> " << *Entry->Scalars[0];
1374 return Str;
1375 }
1376 for (auto V : Entry->Scalars) {
1377 OS << *V;
1378 if (std::any_of(
1379 R->ExternalUses.begin(), R->ExternalUses.end(),
1380 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1381 OS << " <extract>";
1382 OS << "\n";
1383 }
1384 return Str;
1385 }
1386
1387 static std::string getNodeAttributes(const TreeEntry *Entry,
1388 const BoUpSLP *) {
1389 if (Entry->NeedToGather)
1390 return "color=red";
1391 return "";
1392 }
1393};
1394
1395} // end namespace llvm
1396
1397void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1398 ArrayRef<Value *> UserIgnoreLst) {
1399 ExtraValueToDebugLocsMap ExternallyUsedValues;
1400 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1401}
1402
1403void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1404 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1405 ArrayRef<Value *> UserIgnoreLst) {
1406 deleteTree();
1407 UserIgnoreList = UserIgnoreLst;
1408 if (!allSameType(Roots))
1409 return;
1410 buildTree_rec(Roots, 0, -1);
1411
1412 // Collect the values that we need to extract from the tree.
1413 for (TreeEntry &EIdx : VectorizableTree) {
1414 TreeEntry *Entry = &EIdx;
1415
1416 // No need to handle users of gathered values.
1417 if (Entry->NeedToGather)
1418 continue;
1419
1420 // For each lane:
1421 const unsigned Opcode = Entry->State.Opcode;
1422 const unsigned AltOpcode = getAltOpcode(Opcode);
1423 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1424 Value *Scalar = Entry->Scalars[Lane];
1425
1426 if (!sameOpcodeOrAlt(Opcode, AltOpcode,
1427 cast<Instruction>(Scalar)->getOpcode()))
1428 continue;
1429
1430 // Check if the scalar is externally used as an extra arg.
1431 auto ExtI = ExternallyUsedValues.find(Scalar);
1432 if (ExtI != ExternallyUsedValues.end()) {
1433 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
1434 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
1435 ExternalUses.emplace_back(Scalar, nullptr, Lane);
1436 continue;
1437 }
1438 for (User *U : Scalar->users()) {
1439 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
1440
1441 Instruction *UserInst = dyn_cast<Instruction>(U);
1442 if (!UserInst)
1443 continue;
1444
1445 // Skip in-tree scalars that become vectors
1446 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1447 Value *UseScalar = UseEntry->Scalars[0];
1448 // Some in-tree scalars will remain as scalar in vectorized
1449 // instructions. If that is the case, the one in Lane 0 will
1450 // be used.
1451 if (UseScalar != U ||
1452 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1453 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1454 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
1455 assert(!UseEntry->NeedToGather && "Bad state")((!UseEntry->NeedToGather && "Bad state") ? static_cast
<void> (0) : __assert_fail ("!UseEntry->NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1455, __PRETTY_FUNCTION__))
;
1456 continue;
1457 }
1458 }
1459
1460 // Ignore users in the user ignore list.
1461 if (is_contained(UserIgnoreList, UserInst))
1462 continue;
1463
1464 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1465 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
1466 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1467 }
1468 }
1469 }
1470}
1471
1472static Value *getDefaultConstantForOpcode(unsigned Opcode, Type *Ty) {
1473 switch(Opcode) {
1474 case Instruction::Add:
1475 case Instruction::Sub:
1476 case Instruction::Or:
1477 case Instruction::Xor:
1478 case Instruction::Shl:
1479 case Instruction::LShr:
1480 case Instruction::AShr:
1481 return ConstantInt::getNullValue(Ty);
1482 case Instruction::Mul:
1483 case Instruction::UDiv:
1484 case Instruction::SDiv:
1485 case Instruction::URem:
1486 case Instruction::SRem:
1487 return ConstantInt::get(Ty, /*V=*/1);
1488 case Instruction::FAdd:
1489 case Instruction::FSub:
1490 return ConstantFP::get(Ty, /*V=*/0.0);
1491 case Instruction::FMul:
1492 case Instruction::FDiv:
1493 case Instruction::FRem:
1494 return ConstantFP::get(Ty, /*V=*/1.0);
1495 case Instruction::And:
1496 return ConstantInt::getAllOnesValue(Ty);
1497 default:
1498 break;
1499 }
1500 llvm_unreachable("unknown binop for default constant value")::llvm::llvm_unreachable_internal("unknown binop for default constant value"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1500)
;
1501}
1502
1503void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1504 int UserTreeIdx) {
1505 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(((allConstant(VL) || allSameType(VL)) && "Invalid types!"
) ? static_cast<void> (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1505, __PRETTY_FUNCTION__))
;
1506
1507 InstructionsState S = getSameOpcode(VL);
1508 if (Depth == RecursionMaxDepth) {
1509 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
1510 newTreeEntry(VL, false, UserTreeIdx, S);
1511 return;
1512 }
1513
1514 // Don't handle vectors.
1515 if (S.OpValue->getType()->isVectorTy()) {
1516 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
1517 newTreeEntry(VL, false, UserTreeIdx, S);
1518 return;
1519 }
1520
1521 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1522 if (SI->getValueOperand()->getType()->isVectorTy()) {
1523 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1524 newTreeEntry(VL, false, UserTreeIdx, S);
1525 return;
1526 }
1527
1528 // If all of the operands are identical or constant we have a simple solution.
1529 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1530 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1531 newTreeEntry(VL, false, UserTreeIdx, S);
1532 return;
1533 }
1534
1535 // Avoid any vectors that are wider than two elements and
1536 // with real operations less than or equal to half of vector
1537 // to others members are operands to that operations.
1538 unsigned AltOpcode = getAltOpcode(S.Opcode);
1539 unsigned SameOrAlt = 0;
1540 if (VL.size() > 2) {
1541 for (Value *V : VL) {
1542 auto *Instr = cast<Instruction>(V);
1543 if (sameOpcodeOrAlt(S.Opcode, AltOpcode, Instr->getOpcode()))
1544 SameOrAlt++;
1545 }
1546 if (SameOrAlt <= (VL.size() / 2))
1547 return;
1548 }
1549
1550 // We now know that this is a vector of instructions of the same type from
1551 // the same block.
1552
1553 // Don't vectorize ephemeral values.
1554 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1555 if (EphValues.count(VL[i])) {
1556 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1557 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1558 newTreeEntry(VL, false, UserTreeIdx, S);
1559 return;
1560 }
1561 }
1562
1563 // Check if this is a duplicate of another entry.
1564 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1565 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1566 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*VL[i] << ".\n"; } } while (false)
;
1567 if (E->Scalars[i] != VL[i]) {
1568 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1569 newTreeEntry(VL, false, UserTreeIdx, S);
1570 return;
1571 }
1572 }
1573 // Record the reuse of the tree node. FIXME, currently this is only used to
1574 // properly draw the graph rather than for the actual vectorization.
1575 E->UserTreeIndices.push_back(UserTreeIdx);
1576 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
;
1577 return;
1578 }
1579
1580 // Check that none of the instructions in the bundle are already in the tree.
1581 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1582 auto *I = dyn_cast<Instruction>(VL[i]);
1583 if (!I)
1584 continue;
1585 if (getTreeEntry(I)) {
1586 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1587 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1588 newTreeEntry(VL, false, UserTreeIdx, S);
1589 return;
1590 }
1591 }
1592
1593 // If any of the scalars is marked as a value that needs to stay scalar, then
1594 // we need to gather the scalars.
1595 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1596 if (MustGather.count(VL[i])) {
1597 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1598 newTreeEntry(VL, false, UserTreeIdx, S);
1599 return;
1600 }
1601 }
1602
1603 // Check that all of the users of the scalars that we want to vectorize are
1604 // schedulable.
1605 auto *VL0 = cast<Instruction>(S.OpValue);
1606 BasicBlock *BB = VL0->getParent();
1607
1608 if (!DT->isReachableFromEntry(BB)) {
1609 // Don't go into unreachable blocks. They may contain instructions with
1610 // dependency cycles which confuse the final scheduling.
1611 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1612 newTreeEntry(VL, false, UserTreeIdx, S);
1613 return;
1614 }
1615
1616 // Check that every instruction appears once in this bundle.
1617 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1618 for (unsigned j = i + 1; j < e; ++j)
1619 if (VL[i] == VL[j]) {
1620 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1621 newTreeEntry(VL, false, UserTreeIdx, S);
1622 return;
1623 }
1624
1625 auto &BSRef = BlocksSchedules[BB];
1626 if (!BSRef)
1627 BSRef = llvm::make_unique<BlockScheduling>(BB);
1628
1629 BlockScheduling &BS = *BSRef.get();
1630
1631 if (!BS.tryScheduleBundle(VL, this, S.OpValue)) {
1632 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1633 assert((!BS.getScheduleData(VL0) ||(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1635, __PRETTY_FUNCTION__))
1634 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1635, __PRETTY_FUNCTION__))
1635 "tryScheduleBundle should cancelScheduling on failure")(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1635, __PRETTY_FUNCTION__))
;
1636 newTreeEntry(VL, false, UserTreeIdx, S);
1637 return;
1638 }
1639 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1640
1641 unsigned ShuffleOrOp = S.IsAltShuffle ?
1642 (unsigned) Instruction::ShuffleVector : S.Opcode;
1643 switch (ShuffleOrOp) {
1644 case Instruction::PHI: {
1645 PHINode *PH = dyn_cast<PHINode>(VL0);
1646
1647 // Check for terminator values (e.g. invoke).
1648 for (unsigned j = 0; j < VL.size(); ++j)
1649 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1650 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1651 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1652 if (Term) {
1653 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false)
;
1654 BS.cancelScheduling(VL, VL0);
1655 newTreeEntry(VL, false, UserTreeIdx, S);
1656 return;
1657 }
1658 }
1659
1660 newTreeEntry(VL, true, UserTreeIdx, S);
1661 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1662
1663 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1664 ValueList Operands;
1665 // Prepare the operand vector.
1666 for (Value *j : VL)
1667 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1668 PH->getIncomingBlock(i)));
1669
1670 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1671 }
1672 return;
1673 }
1674 case Instruction::ExtractValue:
1675 case Instruction::ExtractElement: {
1676 bool Reuse = canReuseExtract(VL, VL0);
1677 if (Reuse) {
1678 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing extract sequence.\n"
; } } while (false)
;
1679 } else {
1680 BS.cancelScheduling(VL, VL0);
1681 }
1682 newTreeEntry(VL, Reuse, UserTreeIdx, S);
1683 return;
1684 }
1685 case Instruction::Load: {
1686 // Check that a vectorized load would load the same memory as a scalar
1687 // load. For example, we don't want to vectorize loads that are smaller
1688 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1689 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1690 // from such a struct, we read/write packed bits disagreeing with the
1691 // unvectorized version.
1692 Type *ScalarTy = VL0->getType();
1693
1694 if (DL->getTypeSizeInBits(ScalarTy) !=
1695 DL->getTypeAllocSizeInBits(ScalarTy)) {
1696 BS.cancelScheduling(VL, VL0);
1697 newTreeEntry(VL, false, UserTreeIdx, S);
1698 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1699 return;
1700 }
1701
1702 // Make sure all loads in the bundle are simple - we can't vectorize
1703 // atomic or volatile loads.
1704 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1705 LoadInst *L = cast<LoadInst>(VL[i]);
1706 if (!L->isSimple()) {
1707 BS.cancelScheduling(VL, VL0);
1708 newTreeEntry(VL, false, UserTreeIdx, S);
1709 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1710 return;
1711 }
1712 }
1713
1714 // Check if the loads are consecutive, reversed, or neither.
1715 // TODO: What we really want is to sort the loads, but for now, check
1716 // the two likely directions.
1717 bool Consecutive = true;
1718 bool ReverseConsecutive = true;
1719 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1720 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1721 Consecutive = false;
1722 break;
1723 } else {
1724 ReverseConsecutive = false;
1725 }
1726 }
1727
1728 if (Consecutive) {
1729 ++NumLoadsWantToKeepOrder;
1730 newTreeEntry(VL, true, UserTreeIdx, S);
1731 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1732 return;
1733 }
1734
1735 // If none of the load pairs were consecutive when checked in order,
1736 // check the reverse order.
1737 if (ReverseConsecutive)
1738 for (unsigned i = VL.size() - 1; i > 0; --i)
1739 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1740 ReverseConsecutive = false;
1741 break;
1742 }
1743
1744 BS.cancelScheduling(VL, VL0);
1745 newTreeEntry(VL, false, UserTreeIdx, S);
1746
1747 if (ReverseConsecutive) {
1748 ++NumLoadsWantToChangeOrder;
1749 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering reversed loads.\n"
; } } while (false)
;
1750 } else {
1751 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1752 }
1753 return;
1754 }
1755 case Instruction::ZExt:
1756 case Instruction::SExt:
1757 case Instruction::FPToUI:
1758 case Instruction::FPToSI:
1759 case Instruction::FPExt:
1760 case Instruction::PtrToInt:
1761 case Instruction::IntToPtr:
1762 case Instruction::SIToFP:
1763 case Instruction::UIToFP:
1764 case Instruction::Trunc:
1765 case Instruction::FPTrunc:
1766 case Instruction::BitCast: {
1767 Type *SrcTy = VL0->getOperand(0)->getType();
1768 for (unsigned i = 0; i < VL.size(); ++i) {
1769 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1770 if (Ty != SrcTy || !isValidElementType(Ty)) {
1771 BS.cancelScheduling(VL, VL0);
1772 newTreeEntry(VL, false, UserTreeIdx, S);
1773 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1774 return;
1775 }
1776 }
1777 newTreeEntry(VL, true, UserTreeIdx, S);
1778 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1779
1780 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1781 ValueList Operands;
1782 // Prepare the operand vector.
1783 for (Value *j : VL)
1784 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1785
1786 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1787 }
1788 return;
1789 }
1790 case Instruction::ICmp:
1791 case Instruction::FCmp: {
1792 // Check that all of the compares have the same predicate.
1793 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1794 Type *ComparedTy = VL0->getOperand(0)->getType();
1795 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1796 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1797 if (Cmp->getPredicate() != P0 ||
1798 Cmp->getOperand(0)->getType() != ComparedTy) {
1799 BS.cancelScheduling(VL, VL0);
1800 newTreeEntry(VL, false, UserTreeIdx, S);
1801 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1802 return;
1803 }
1804 }
1805
1806 newTreeEntry(VL, true, UserTreeIdx, S);
1807 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1808
1809 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1810 ValueList Operands;
1811 // Prepare the operand vector.
1812 for (Value *j : VL)
1813 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1814
1815 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1816 }
1817 return;
1818 }
1819 case Instruction::Select:
1820 case Instruction::Add:
1821 case Instruction::FAdd:
1822 case Instruction::Sub:
1823 case Instruction::FSub:
1824 case Instruction::Mul:
1825 case Instruction::FMul:
1826 case Instruction::UDiv:
1827 case Instruction::SDiv:
1828 case Instruction::FDiv:
1829 case Instruction::URem:
1830 case Instruction::SRem:
1831 case Instruction::FRem:
1832 case Instruction::Shl:
1833 case Instruction::LShr:
1834 case Instruction::AShr:
1835 case Instruction::And:
1836 case Instruction::Or:
1837 case Instruction::Xor:
1838 newTreeEntry(VL, true, UserTreeIdx, S);
1839 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1840
1841 // Sort operands of the instructions so that each side is more likely to
1842 // have the same opcode.
1843 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1844 ValueList Left, Right;
1845 reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1846 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1847 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1848 return;
1849 }
1850
1851 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1852 ValueList Operands;
1853 // Prepare the operand vector.
1854 for (Value *VecOp : VL) {
1855 auto *I = cast<Instruction>(VecOp);
1856 if (I->getOpcode() == S.Opcode) {
1857 Operands.push_back(I->getOperand(i));
1858 continue;
1859 }
1860 assert(Instruction::isBinaryOp(S.Opcode) &&((Instruction::isBinaryOp(S.Opcode) && "Expected a binary operation."
) ? static_cast<void> (0) : __assert_fail ("Instruction::isBinaryOp(S.Opcode) && \"Expected a binary operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1861, __PRETTY_FUNCTION__))
1861 "Expected a binary operation.")((Instruction::isBinaryOp(S.Opcode) && "Expected a binary operation."
) ? static_cast<void> (0) : __assert_fail ("Instruction::isBinaryOp(S.Opcode) && \"Expected a binary operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1861, __PRETTY_FUNCTION__))
;
1862 Value *Operand = isOdd(i)
1863 ? getDefaultConstantForOpcode(S.Opcode, I->getType())
1864 : VecOp;
1865 Operands.push_back(Operand);
1866 }
1867 if (allSameType(Operands))
1868 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1869 }
1870 return;
1871
1872 case Instruction::GetElementPtr: {
1873 // We don't combine GEPs with complicated (nested) indexing.
1874 for (unsigned j = 0; j < VL.size(); ++j) {
1875 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1876 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1877 BS.cancelScheduling(VL, VL0);
1878 newTreeEntry(VL, false, UserTreeIdx, S);
1879 return;
1880 }
1881 }
1882
1883 // We can't combine several GEPs into one vector if they operate on
1884 // different types.
1885 Type *Ty0 = VL0->getOperand(0)->getType();
1886 for (unsigned j = 0; j < VL.size(); ++j) {
1887 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1888 if (Ty0 != CurTy) {
1889 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1890 BS.cancelScheduling(VL, VL0);
1891 newTreeEntry(VL, false, UserTreeIdx, S);
1892 return;
1893 }
1894 }
1895
1896 // We don't combine GEPs with non-constant indexes.
1897 for (unsigned j = 0; j < VL.size(); ++j) {
1898 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1899 if (!isa<ConstantInt>(Op)) {
1900 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1901 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1902 BS.cancelScheduling(VL, VL0);
1903 newTreeEntry(VL, false, UserTreeIdx, S);
1904 return;
1905 }
1906 }
1907
1908 newTreeEntry(VL, true, UserTreeIdx, S);
1909 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1910 for (unsigned i = 0, e = 2; i < e; ++i) {
1911 ValueList Operands;
1912 // Prepare the operand vector.
1913 for (Value *j : VL)
1914 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1915
1916 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1917 }
1918 return;
1919 }
1920 case Instruction::Store: {
1921 // Check if the stores are consecutive or of we need to swizzle them.
1922 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1923 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1924 BS.cancelScheduling(VL, VL0);
1925 newTreeEntry(VL, false, UserTreeIdx, S);
1926 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1927 return;
1928 }
1929
1930 newTreeEntry(VL, true, UserTreeIdx, S);
1931 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1932
1933 ValueList Operands;
1934 for (Value *j : VL)
1935 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1936
1937 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1938 return;
1939 }
1940 case Instruction::Call: {
1941 // Check if the calls are all to the same vectorizable intrinsic.
1942 CallInst *CI = cast<CallInst>(VL0);
1943 // Check if this is an Intrinsic call or something that can be
1944 // represented by an intrinsic call
1945 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1946 if (!isTriviallyVectorizable(ID)) {
1947 BS.cancelScheduling(VL, VL0);
1948 newTreeEntry(VL, false, UserTreeIdx, S);
1949 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1950 return;
1951 }
1952 Function *Int = CI->getCalledFunction();
1953 Value *A1I = nullptr;
1954 if (hasVectorInstrinsicScalarOpd(ID, 1))
1955 A1I = CI->getArgOperand(1);
1956 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1957 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1958 if (!CI2 || CI2->getCalledFunction() != Int ||
1959 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1960 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1961 BS.cancelScheduling(VL, VL0);
1962 newTreeEntry(VL, false, UserTreeIdx, S);
1963 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1964 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1965 return;
1966 }
1967 // ctlz,cttz and powi are special intrinsics whose second argument
1968 // should be same in order for them to be vectorized.
1969 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1970 Value *A1J = CI2->getArgOperand(1);
1971 if (A1I != A1J) {
1972 BS.cancelScheduling(VL, VL0);
1973 newTreeEntry(VL, false, UserTreeIdx, S);
1974 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1975 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1976 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
;
1977 return;
1978 }
1979 }
1980 // Verify that the bundle operands are identical between the two calls.
1981 if (CI->hasOperandBundles() &&
1982 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1983 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1984 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1985 BS.cancelScheduling(VL, VL0);
1986 newTreeEntry(VL, false, UserTreeIdx, S);
1987 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1988 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1989 return;
1990 }
1991 }
1992
1993 newTreeEntry(VL, true, UserTreeIdx, S);
1994 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1995 ValueList Operands;
1996 // Prepare the operand vector.
1997 for (Value *j : VL) {
1998 CallInst *CI2 = dyn_cast<CallInst>(j);
1999 Operands.push_back(CI2->getArgOperand(i));
2000 }
2001 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
2002 }
2003 return;
2004 }
2005 case Instruction::ShuffleVector:
2006 // If this is not an alternate sequence of opcode like add-sub
2007 // then do not vectorize this instruction.
2008 if (!S.IsAltShuffle) {
2009 BS.cancelScheduling(VL, VL0);
2010 newTreeEntry(VL, false, UserTreeIdx, S);
2011 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
2012 return;
2013 }
2014 newTreeEntry(VL, true, UserTreeIdx, S);
2015 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
2016
2017 // Reorder operands if reordering would enable vectorization.
2018 if (isa<BinaryOperator>(VL0)) {
2019 ValueList Left, Right;
2020 reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
2021 buildTree_rec(Left, Depth + 1, UserTreeIdx);
2022 buildTree_rec(Right, Depth + 1, UserTreeIdx);
2023 return;
2024 }
2025
2026 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
2027 ValueList Operands;
2028 // Prepare the operand vector.
2029 for (Value *VecOp : VL) {
2030 auto *I = cast<Instruction>(VecOp);
2031 if (sameOpcodeOrAlt(S.Opcode, AltOpcode, I->getOpcode())) {
2032 Operands.push_back(I->getOperand(i));
2033 continue;
2034 }
2035 assert(Instruction::isBinaryOp(S.Opcode) &&((Instruction::isBinaryOp(S.Opcode) && "Expected a binary operation."
) ? static_cast<void> (0) : __assert_fail ("Instruction::isBinaryOp(S.Opcode) && \"Expected a binary operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2036, __PRETTY_FUNCTION__))
2036 "Expected a binary operation.")((Instruction::isBinaryOp(S.Opcode) && "Expected a binary operation."
) ? static_cast<void> (0) : __assert_fail ("Instruction::isBinaryOp(S.Opcode) && \"Expected a binary operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2036, __PRETTY_FUNCTION__))
;
2037 Value *Operand = isOdd(i)
2038 ? getDefaultConstantForOpcode(S.Opcode, I->getType())
2039 : VecOp;
2040 Operands.push_back(Operand);
2041 }
2042
2043 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
2044 }
2045 return;
2046
2047 default:
2048 BS.cancelScheduling(VL, VL0);
2049 newTreeEntry(VL, false, UserTreeIdx, S);
2050 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
2051 return;
2052 }
2053}
2054
2055unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
2056 unsigned N;
2057 Type *EltTy;
2058 auto *ST = dyn_cast<StructType>(T);
2059 if (ST) {
2060 N = ST->getNumElements();
2061 EltTy = *ST->element_begin();
2062 } else {
2063 N = cast<ArrayType>(T)->getNumElements();
2064 EltTy = cast<ArrayType>(T)->getElementType();
2065 }
2066 if (!isValidElementType(EltTy))
2067 return 0;
2068 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
2069 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
2070 return 0;
2071 if (ST) {
2072 // Check that struct is homogeneous.
2073 for (const auto *Ty : ST->elements())
2074 if (Ty != EltTy)
2075 return 0;
2076 }
2077 return N;
2078}
2079
2080bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const {
2081 Instruction *E0 = cast<Instruction>(OpValue);
2082 assert(E0->getOpcode() == Instruction::ExtractElement ||((E0->getOpcode() == Instruction::ExtractElement || E0->
getOpcode() == Instruction::ExtractValue) ? static_cast<void
> (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2083, __PRETTY_FUNCTION__))
2083 E0->getOpcode() == Instruction::ExtractValue)((E0->getOpcode() == Instruction::ExtractElement || E0->
getOpcode() == Instruction::ExtractValue) ? static_cast<void
> (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2083, __PRETTY_FUNCTION__))
;
2084 assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode")((E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode"
) ? static_cast<void> (0) : __assert_fail ("E0->getOpcode() == getSameOpcode(VL).Opcode && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2084, __PRETTY_FUNCTION__))
;
2085 // Check if all of the extracts come from the same vector and from the
2086 // correct offset.
2087 Value *Vec = E0->getOperand(0);
2088
2089 // We have to extract from a vector/aggregate with the same number of elements.
2090 unsigned NElts;
2091 if (E0->getOpcode() == Instruction::ExtractValue) {
2092 const DataLayout &DL = E0->getModule()->getDataLayout();
2093 NElts = canMapToVector(Vec->getType(), DL);
2094 if (!NElts)
2095 return false;
2096 // Check if load can be rewritten as load of vector.
2097 LoadInst *LI = dyn_cast<LoadInst>(Vec);
2098 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2099 return false;
2100 } else {
2101 NElts = Vec->getType()->getVectorNumElements();
2102 }
2103
2104 if (NElts != VL.size())
2105 return false;
2106
2107 // Check that all of the indices extract from the correct offset.
2108 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
2109 Instruction *Inst = cast<Instruction>(VL[I]);
2110 if (!matchExtractIndex(Inst, I, Inst->getOpcode()))
2111 return false;
2112 if (Inst->getOperand(0) != Vec)
2113 return false;
2114 }
2115
2116 return true;
2117}
2118
2119bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2120 return I->hasOneUse() ||
2121 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2122 return ScalarToTreeEntry.count(U) > 0;
2123 });
2124}
2125
2126int BoUpSLP::getEntryCost(TreeEntry *E) {
2127 ArrayRef<Value*> VL = E->Scalars;
2128
2129 Type *ScalarTy = VL[0]->getType();
2130 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2131 ScalarTy = SI->getValueOperand()->getType();
2132 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2133 ScalarTy = CI->getOperand(0)->getType();
2134 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2135
2136 // If we have computed a smaller type for the expression, update VecTy so
2137 // that the costs will be accurate.
2138 if (MinBWs.count(VL[0]))
2139 VecTy = VectorType::get(
2140 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2141
2142 if (E->NeedToGather) {
2143 if (allConstant(VL))
2144 return 0;
2145 if (isSplat(VL)) {
2146 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2147 }
2148 if (getSameOpcode(VL).Opcode == Instruction::ExtractElement) {
2149 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
2150 if (ShuffleKind.hasValue()) {
2151 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2152 for (auto *V : VL) {
2153 // If all users of instruction are going to be vectorized and this
2154 // instruction itself is not going to be vectorized, consider this
2155 // instruction as dead and remove its cost from the final cost of the
2156 // vectorized tree.
2157 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2158 !ScalarToTreeEntry.count(V)) {
2159 auto *IO = cast<ConstantInt>(
2160 cast<ExtractElementInst>(V)->getIndexOperand());
2161 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2162 IO->getZExtValue());
2163 }
2164 }
2165 return Cost;
2166 }
2167 }
2168 return getGatherCost(E->Scalars);
2169 }
2170 assert(E->State.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")((E->State.Opcode && allSameType(VL) && allSameBlock
(VL) && "Invalid VL") ? static_cast<void> (0) :
__assert_fail ("E->State.Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2170, __PRETTY_FUNCTION__))
;
2171 auto *VL0 = cast<Instruction>(E->State.OpValue);
2172 unsigned ShuffleOrOp = E->State.IsAltShuffle ?
2173 (unsigned) Instruction::ShuffleVector : E->State.Opcode;
2174 switch (ShuffleOrOp) {
2175 case Instruction::PHI:
2176 return 0;
2177
2178 case Instruction::ExtractValue:
2179 case Instruction::ExtractElement:
2180 if (canReuseExtract(VL, E->State.OpValue)) {
2181 int DeadCost = 0;
2182 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2183 Instruction *E = cast<Instruction>(VL[i]);
2184 // If all users are going to be vectorized, instruction can be
2185 // considered as dead.
2186 // The same, if have only one user, it will be vectorized for sure.
2187 if (areAllUsersVectorized(E))
2188 // Take credit for instruction that will become dead.
2189 DeadCost +=
2190 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2191 }
2192 return -DeadCost;
2193 }
2194 return getGatherCost(VecTy);
2195
2196 case Instruction::ZExt:
2197 case Instruction::SExt:
2198 case Instruction::FPToUI:
2199 case Instruction::FPToSI:
2200 case Instruction::FPExt:
2201 case Instruction::PtrToInt:
2202 case Instruction::IntToPtr:
2203 case Instruction::SIToFP:
2204 case Instruction::UIToFP:
2205 case Instruction::Trunc:
2206 case Instruction::FPTrunc:
2207 case Instruction::BitCast: {
2208 Type *SrcTy = VL0->getOperand(0)->getType();
2209
2210 // Calculate the cost of this instruction.
2211 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2212 VL0->getType(), SrcTy, VL0);
2213
2214 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2215 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2216 return VecCost - ScalarCost;
2217 }
2218 case Instruction::FCmp:
2219 case Instruction::ICmp:
2220 case Instruction::Select: {
2221 // Calculate the cost of this instruction.
2222 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2223 int ScalarCost = VecTy->getNumElements() *
2224 TTI->getCmpSelInstrCost(ShuffleOrOp, ScalarTy, Builder.getInt1Ty(), VL0);
2225 int VecCost = TTI->getCmpSelInstrCost(ShuffleOrOp, VecTy, MaskTy, VL0);
2226 return VecCost - ScalarCost;
2227 }
2228 case Instruction::Add:
2229 case Instruction::FAdd:
2230 case Instruction::Sub:
2231 case Instruction::FSub:
2232 case Instruction::Mul:
2233 case Instruction::FMul:
2234 case Instruction::UDiv:
2235 case Instruction::SDiv:
2236 case Instruction::FDiv:
2237 case Instruction::URem:
2238 case Instruction::SRem:
2239 case Instruction::FRem:
2240 case Instruction::Shl:
2241 case Instruction::LShr:
2242 case Instruction::AShr:
2243 case Instruction::And:
2244 case Instruction::Or:
2245 case Instruction::Xor: {
2246 // Certain instructions can be cheaper to vectorize if they have a
2247 // constant second vector operand.
2248 TargetTransformInfo::OperandValueKind Op1VK =
2249 TargetTransformInfo::OK_AnyValue;
2250 TargetTransformInfo::OperandValueKind Op2VK =
2251 TargetTransformInfo::OK_AnyValue;
2252 TargetTransformInfo::OperandValueProperties Op1VP =
2253 TargetTransformInfo::OP_None;
2254 TargetTransformInfo::OperandValueProperties Op2VP =
2255 TargetTransformInfo::OP_None;
2256
2257 // If all operands are exactly the same ConstantInt then set the
2258 // operand kind to OK_UniformConstantValue.
2259 // If instead not all operands are constants, then set the operand kind
2260 // to OK_AnyValue. If all operands are constants but not the same,
2261 // then set the operand kind to OK_NonUniformConstantValue.
2262 if (auto *CInt = dyn_cast<ConstantInt>(VL0->getOperand(1))) {
2263 Op2VK = TargetTransformInfo::OK_UniformConstantValue;
2264 const unsigned Opcode = E->State.Opcode;
2265 for (auto *V : VL) {
2266 auto *I = cast<Instruction>(V);
2267 if (I == VL0 || Opcode != I->getOpcode())
2268 continue;
2269 if (!isa<ConstantInt>(I->getOperand(1))) {
2270 Op2VK = TargetTransformInfo::OK_AnyValue;
2271 break;
2272 }
2273 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
2274 CInt != cast<ConstantInt>(I->getOperand(1)))
2275 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2276 }
2277 // FIXME: Currently cost of model modification for division by power of
2278 // 2 is handled for X86 and AArch64. Add support for other targets.
2279 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
2280 CInt->getValue().isPowerOf2())
2281 Op2VP = TargetTransformInfo::OP_PowerOf2;
2282 }
2283
2284 int ScalarCost = VecTy->getNumElements() *
2285 TTI->getArithmeticInstrCost(E->State.Opcode, ScalarTy,
2286 Op1VK, Op2VK, Op1VP, Op2VP);
2287 int VecCost = TTI->getArithmeticInstrCost(E->State.Opcode, VecTy, Op1VK,
2288 Op2VK, Op1VP, Op2VP);
2289 return VecCost - ScalarCost;
2290 }
2291 case Instruction::GetElementPtr: {
2292 TargetTransformInfo::OperandValueKind Op1VK =
2293 TargetTransformInfo::OK_AnyValue;
2294 TargetTransformInfo::OperandValueKind Op2VK =
2295 TargetTransformInfo::OK_UniformConstantValue;
2296
2297 int ScalarCost =
2298 VecTy->getNumElements() *
2299 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2300 int VecCost =
2301 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2302
2303 return VecCost - ScalarCost;
2304 }
2305 case Instruction::Load: {
2306 // Cost of wide load - cost of scalar loads.
2307 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2308 int ScalarLdCost = VecTy->getNumElements() *
2309 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2310 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2311 VecTy, alignment, 0, VL0);
2312 return VecLdCost - ScalarLdCost;
2313 }
2314 case Instruction::Store: {
2315 // We know that we can merge the stores. Calculate the cost.
2316 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2317 int ScalarStCost = VecTy->getNumElements() *
2318 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2319 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2320 VecTy, alignment, 0, VL0);
2321 return VecStCost - ScalarStCost;
2322 }
2323 case Instruction::Call: {
2324 CallInst *CI = cast<CallInst>(VL0);
2325 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2326
2327 // Calculate the cost of the scalar and vector calls.
2328 SmallVector<Type*, 4> ScalarTys;
2329 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2330 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2331
2332 FastMathFlags FMF;
2333 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2334 FMF = FPMO->getFastMathFlags();
2335
2336 int ScalarCallCost = VecTy->getNumElements() *
2337 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2338
2339 SmallVector<Value *, 4> Args(CI->arg_operands());
2340 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2341 VecTy->getNumElements());
2342
2343 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2344 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2345 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
2346
2347 return VecCallCost - ScalarCallCost;
2348 }
2349 case Instruction::ShuffleVector: {
2350 TargetTransformInfo::OperandValueKind Op1VK =
2351 TargetTransformInfo::OK_AnyValue;
2352 TargetTransformInfo::OperandValueKind Op2VK =
2353 TargetTransformInfo::OK_AnyValue;
2354 unsigned AltOpcode = getAltOpcode(E->State.Opcode);
2355 int ScalarCost =
2356 TTI->getArithmeticInstrCost(E->State.Opcode, ScalarTy, Op1VK, Op2VK) *
2357 VL.size() / 2;
2358 ScalarCost +=
2359 TTI->getArithmeticInstrCost(AltOpcode, ScalarTy, Op1VK, Op2VK) *
2360 VL.size() / 2;
2361 // VecCost is equal to sum of the cost of creating 2 vectors
2362 // and the cost of creating shuffle.
2363 int VecCost =
2364 TTI->getArithmeticInstrCost(E->State.Opcode, VecTy, Op1VK, Op2VK);
2365 VecCost += TTI->getArithmeticInstrCost(AltOpcode, VecTy, Op1VK, Op2VK);
2366 VecCost +=
2367 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2368 return VecCost - ScalarCost;
2369 }
2370 default:
2371 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2371)
;
2372 }
2373}
2374
2375bool BoUpSLP::isFullyVectorizableTinyTree() {
2376 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
2377 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
2378
2379 // We only handle trees of heights 1 and 2.
2380 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2381 return true;
2382
2383 if (VectorizableTree.size() != 2)
2384 return false;
2385
2386 // Handle splat and all-constants stores.
2387 if (!VectorizableTree[0].NeedToGather &&
2388 (allConstant(VectorizableTree[1].Scalars) ||
2389 isSplat(VectorizableTree[1].Scalars)))
2390 return true;
2391
2392 // Gathering cost would be too much for tiny trees.
2393 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2394 return false;
2395
2396 return true;
2397}
2398
2399bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2400 // We can vectorize the tree if its size is greater than or equal to the
2401 // minimum size specified by the MinTreeSize command line option.
2402 if (VectorizableTree.size() >= MinTreeSize)
2403 return false;
2404
2405 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2406 // can vectorize it if we can prove it fully vectorizable.
2407 if (isFullyVectorizableTinyTree())
2408 return false;
2409
2410 assert(VectorizableTree.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2412, __PRETTY_FUNCTION__))
2411 ? ExternalUses.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2412, __PRETTY_FUNCTION__))
2412 : true && "We shouldn't have any external users")((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2412, __PRETTY_FUNCTION__))
;
2413
2414 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2415 // vectorizable.
2416 return true;
2417}
2418
2419int BoUpSLP::getSpillCost() {
2420 // Walk from the bottom of the tree to the top, tracking which values are
2421 // live. When we see a call instruction that is not part of our tree,
2422 // query TTI to see if there is a cost to keeping values live over it
2423 // (for example, if spills and fills are required).
2424 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2425 int Cost = 0;
2426
2427 SmallPtrSet<Instruction*, 4> LiveValues;
2428 Instruction *PrevInst = nullptr;
2429
2430 for (const auto &N : VectorizableTree) {
2431 Instruction *Inst = dyn_cast<Instruction>(N.State.OpValue);
2432 if (!Inst)
2433 continue;
2434
2435 if (!PrevInst) {
2436 PrevInst = Inst;
2437 continue;
2438 }
2439
2440 // Update LiveValues.
2441 LiveValues.erase(PrevInst);
2442 for (auto &J : PrevInst->operands()) {
2443 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2444 LiveValues.insert(cast<Instruction>(&*J));
2445 }
2446
2447 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2448 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2449 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2450 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2451 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2452 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2453 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
;
2454
2455 // Now find the sequence of instructions between PrevInst and Inst.
2456 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2457 PrevInstIt =
2458 PrevInst->getIterator().getReverse();
2459 while (InstIt != PrevInstIt) {
2460 if (PrevInstIt == PrevInst->getParent()->rend()) {
2461 PrevInstIt = Inst->getParent()->rbegin();
2462 continue;
2463 }
2464
2465 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2466 SmallVector<Type*, 4> V;
2467 for (auto *II : LiveValues)
2468 V.push_back(VectorType::get(II->getType(), BundleWidth));
2469 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2470 }
2471
2472 ++PrevInstIt;
2473 }
2474
2475 PrevInst = Inst;
2476 }
2477
2478 return Cost;
2479}
2480
2481int BoUpSLP::getTreeCost() {
2482 int Cost = 0;
2483 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
2484 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
2485
2486 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2487
2488 for (TreeEntry &TE : VectorizableTree) {
2489 int C = getEntryCost(&TE);
2490 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.State.OpValue <<
".\n"; } } while (false)
2491 << *TE.State.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.State.OpValue <<
".\n"; } } while (false)
;
2492 Cost += C;
2493 }
2494
2495 SmallSet<Value *, 16> ExtractCostCalculated;
2496 int ExtractCost = 0;
2497 for (ExternalUser &EU : ExternalUses) {
2498 // We only add extract cost once for the same scalar.
2499 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2500 continue;
2501
2502 // Uses by ephemeral values are free (because the ephemeral value will be
2503 // removed prior to code generation, and so the extraction will be
2504 // removed as well).
2505 if (EphValues.count(EU.User))
2506 continue;
2507
2508 // If we plan to rewrite the tree in a smaller type, we will need to sign
2509 // extend the extracted value back to the original type. Here, we account
2510 // for the extract and the added cost of the sign extend if needed.
2511 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2512 auto *ScalarRoot = VectorizableTree[0].State.OpValue;
2513 if (MinBWs.count(ScalarRoot)) {
2514 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2515 auto Extend =
2516 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2517 VecTy = VectorType::get(MinTy, BundleWidth);
2518 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2519 VecTy, EU.Lane);
2520 } else {
2521 ExtractCost +=
2522 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2523 }
2524 }
2525
2526 int SpillCost = getSpillCost();
2527 Cost += SpillCost + ExtractCost;
2528
2529 std::string Str;
2530 {
2531 raw_string_ostream OS(Str);
2532 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2533 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2534 << "SLP: Total Cost = " << Cost << ".\n";
2535 }
2536 DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
2537
2538 if (ViewSLPTree)
2539 ViewGraph(this, "SLP" + F->getName(), false, Str);
2540
2541 return Cost;
2542}
2543
2544int BoUpSLP::getGatherCost(Type *Ty) {
2545 int Cost = 0;
2546 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2547 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2548 return Cost;
2549}
2550
2551int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2552 // Find the type of the operands in VL.
2553 Type *ScalarTy = VL[0]->getType();
2554 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2555 ScalarTy = SI->getValueOperand()->getType();
2556 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2557 // Find the cost of inserting/extracting values from the vector.
2558 return getGatherCost(VecTy);
2559}
2560
2561// Reorder commutative operations in alternate shuffle if the resulting vectors
2562// are consecutive loads. This would allow us to vectorize the tree.
2563// If we have something like-
2564// load a[0] - load b[0]
2565// load b[1] + load a[1]
2566// load a[2] - load b[2]
2567// load a[3] + load b[3]
2568// Reordering the second load b[1] load a[1] would allow us to vectorize this
2569// code.
2570void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2571 SmallVectorImpl<Value *> &Left,
2572 SmallVectorImpl<Value *> &Right) {
2573 // Push left and right operands of binary operation into Left and Right
2574 unsigned AltOpcode = getAltOpcode(Opcode);
2575 for (Value *V : VL) {
2576 auto *I = cast<Instruction>(V);
2577 if (sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode())) {
2578 Left.push_back(I->getOperand(0));
2579 Right.push_back(I->getOperand(1));
2580 } else {
2581 Left.push_back(I);
2582 Right.push_back(getDefaultConstantForOpcode(Opcode, I->getType()));
2583 }
2584 }
2585
2586 // Reorder if we have a commutative operation and consecutive access
2587 // are on either side of the alternate instructions.
2588 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2589 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2590 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2591 Instruction *VL1 = cast<Instruction>(VL[j]);
2592 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2593 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2594 std::swap(Left[j], Right[j]);
2595 continue;
2596 } else if (VL2->isCommutative() &&
2597 isConsecutiveAccess(L, L1, *DL, *SE)) {
2598 std::swap(Left[j + 1], Right[j + 1]);
2599 continue;
2600 }
2601 // else unchanged
2602 }
2603 }
2604 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2605 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2606 Instruction *VL1 = cast<Instruction>(VL[j]);
2607 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2608 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2609 std::swap(Left[j], Right[j]);
2610 continue;
2611 } else if (VL2->isCommutative() &&
2612 isConsecutiveAccess(L, L1, *DL, *SE)) {
2613 std::swap(Left[j + 1], Right[j + 1]);
2614 continue;
2615 }
2616 // else unchanged
2617 }
2618 }
2619 }
2620}
2621
2622// Return true if I should be commuted before adding it's left and right
2623// operands to the arrays Left and Right.
2624//
2625// The vectorizer is trying to either have all elements one side being
2626// instruction with the same opcode to enable further vectorization, or having
2627// a splat to lower the vectorizing cost.
2628static bool shouldReorderOperands(
2629 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2630 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2631 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2632 if (I.getOpcode() == Opcode) {
2633 VLeft = I.getOperand(0);
2634 VRight = I.getOperand(1);
2635 } else {
2636 VLeft = &I;
2637 VRight = getDefaultConstantForOpcode(Opcode, I.getType());
2638 }
2639 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2640 if (SplatRight) {
2641 if (VRight == Right[i - 1])
2642 // Preserve SplatRight
2643 return false;
2644 if (VLeft == Right[i - 1]) {
2645 // Commuting would preserve SplatRight, but we don't want to break
2646 // SplatLeft either, i.e. preserve the original order if possible.
2647 // (FIXME: why do we care?)
2648 if (SplatLeft && VLeft == Left[i - 1])
2649 return false;
2650 return true;
2651 }
2652 }
2653 // Symmetrically handle Right side.
2654 if (SplatLeft) {
2655 if (VLeft == Left[i - 1])
2656 // Preserve SplatLeft
2657 return false;
2658 if (VRight == Left[i - 1])
2659 return true;
2660 }
2661
2662 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2663 Instruction *IRight = dyn_cast<Instruction>(VRight);
2664
2665 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2666 // it and not the right, in this case we want to commute.
2667 if (AllSameOpcodeRight) {
2668 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2669 if (IRight && RightPrevOpcode == IRight->getOpcode())
2670 // Do not commute, a match on the right preserves AllSameOpcodeRight
2671 return false;
2672 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2673 // We have a match and may want to commute, but first check if there is
2674 // not also a match on the existing operands on the Left to preserve
2675 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2676 // (FIXME: why do we care?)
2677 if (AllSameOpcodeLeft && ILeft &&
2678 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2679 return false;
2680 return true;
2681 }
2682 }
2683 // Symmetrically handle Left side.
2684 if (AllSameOpcodeLeft) {
2685 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2686 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2687 return false;
2688 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2689 return true;
2690 }
2691 return false;
2692}
2693
2694void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2695 ArrayRef<Value *> VL,
2696 SmallVectorImpl<Value *> &Left,
2697 SmallVectorImpl<Value *> &Right) {
2698 if (!VL.empty()) {
2699 // Peel the first iteration out of the loop since there's nothing
2700 // interesting to do anyway and it simplifies the checks in the loop.
2701 auto *I = cast<Instruction>(VL[0]);
2702 Value *VLeft;
2703 Value *VRight;
2704 if (I->getOpcode() == Opcode) {
2705 VLeft = I->getOperand(0);
2706 VRight = I->getOperand(1);
2707 } else {
2708 VLeft = I;
2709 VRight = getDefaultConstantForOpcode(Opcode, I->getType());
2710 }
2711 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2712 // Favor having instruction to the right. FIXME: why?
2713 std::swap(VLeft, VRight);
2714 Left.push_back(VLeft);
2715 Right.push_back(VRight);
2716 }
2717
2718 // Keep track if we have instructions with all the same opcode on one side.
2719 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2720 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2721 // Keep track if we have one side with all the same value (broadcast).
2722 bool SplatLeft = true;
2723 bool SplatRight = true;
2724
2725 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2726 Instruction *I = cast<Instruction>(VL[i]);
2727 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||((((I->getOpcode() == Opcode && I->isCommutative
()) || (I->getOpcode() != Opcode && Instruction::isCommutative
(Opcode))) && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2729, __PRETTY_FUNCTION__))
2728 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&((((I->getOpcode() == Opcode && I->isCommutative
()) || (I->getOpcode() != Opcode && Instruction::isCommutative
(Opcode))) && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2729, __PRETTY_FUNCTION__))
2729 "Can only process commutative instruction")((((I->getOpcode() == Opcode && I->isCommutative
()) || (I->getOpcode() != Opcode && Instruction::isCommutative
(Opcode))) && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2729, __PRETTY_FUNCTION__))
;
2730 // Commute to favor either a splat or maximizing having the same opcodes on
2731 // one side.
2732 Value *VLeft;
2733 Value *VRight;
2734 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2735 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2736 VRight)) {
2737 Left.push_back(VRight);
2738 Right.push_back(VLeft);
2739 } else {
2740 Left.push_back(VLeft);
2741 Right.push_back(VRight);
2742 }
2743 // Update Splat* and AllSameOpcode* after the insertion.
2744 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2745 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2746 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2747 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2748 cast<Instruction>(Left[i])->getOpcode());
2749 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2750 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2751 cast<Instruction>(Right[i])->getOpcode());
2752 }
2753
2754 // If one operand end up being broadcast, return this operand order.
2755 if (SplatRight || SplatLeft)
2756 return;
2757
2758 // Finally check if we can get longer vectorizable chain by reordering
2759 // without breaking the good operand order detected above.
2760 // E.g. If we have something like-
2761 // load a[0] load b[0]
2762 // load b[1] load a[1]
2763 // load a[2] load b[2]
2764 // load a[3] load b[3]
2765 // Reordering the second load b[1] load a[1] would allow us to vectorize
2766 // this code and we still retain AllSameOpcode property.
2767 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2768 // such as-
2769 // add a[0],c[0] load b[0]
2770 // add a[1],c[2] load b[1]
2771 // b[2] load b[2]
2772 // add a[3],c[3] load b[3]
2773 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2774 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2775 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2776 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2777 std::swap(Left[j + 1], Right[j + 1]);
2778 continue;
2779 }
2780 }
2781 }
2782 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2783 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2784 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2785 std::swap(Left[j + 1], Right[j + 1]);
2786 continue;
2787 }
2788 }
2789 }
2790 // else unchanged
2791 }
2792}
2793
2794void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2795 // Get the basic block this bundle is in. All instructions in the bundle
2796 // should be in this block.
2797 auto *Front = cast<Instruction>(OpValue);
2798 auto *BB = Front->getParent();
2799 const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2800 const unsigned AltOpcode = getAltOpcode(Opcode);
2801 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast
<Instruction>(V)->getOpcode()) || cast<Instruction
>(V)->getParent() == BB; })) ? static_cast<void> (
0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2805, __PRETTY_FUNCTION__))
2802 return !sameOpcodeOrAlt(Opcode, AltOpcode,((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast
<Instruction>(V)->getOpcode()) || cast<Instruction
>(V)->getParent() == BB; })) ? static_cast<void> (
0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2805, __PRETTY_FUNCTION__))
2803 cast<Instruction>(V)->getOpcode()) ||((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast
<Instruction>(V)->getOpcode()) || cast<Instruction
>(V)->getParent() == BB; })) ? static_cast<void> (
0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2805, __PRETTY_FUNCTION__))
2804 cast<Instruction>(V)->getParent() == BB;((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast
<Instruction>(V)->getOpcode()) || cast<Instruction
>(V)->getParent() == BB; })) ? static_cast<void> (
0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2805, __PRETTY_FUNCTION__))
2805 }))((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast
<Instruction>(V)->getOpcode()) || cast<Instruction
>(V)->getParent() == BB; })) ? static_cast<void> (
0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2805, __PRETTY_FUNCTION__))
;
2806
2807 // The last instruction in the bundle in program order.
2808 Instruction *LastInst = nullptr;
10
'LastInst' initialized to a null pointer value
2809
2810 // Find the last instruction. The common case should be that BB has been
2811 // scheduled, and the last instruction is VL.back(). So we start with
2812 // VL.back() and iterate over schedule data until we reach the end of the
2813 // bundle. The end of the bundle is marked by null ScheduleData.
2814 if (BlocksSchedules.count(BB)) {
11
Assuming the condition is false
12
Taking false branch
2815 auto *Bundle =
2816 BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2817 if (Bundle && Bundle->isPartOfBundle())
2818 for (; Bundle; Bundle = Bundle->NextInBundle)
2819 if (Bundle->OpValue == Bundle->Inst)
2820 LastInst = Bundle->Inst;
2821 }
2822
2823 // LastInst can still be null at this point if there's either not an entry
2824 // for BB in BlocksSchedules or there's no ScheduleData available for
2825 // VL.back(). This can be the case if buildTree_rec aborts for various
2826 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2827 // size is reached, etc.). ScheduleData is initialized in the scheduling
2828 // "dry-run".
2829 //
2830 // If this happens, we can still find the last instruction by brute force. We
2831 // iterate forwards from Front (inclusive) until we either see all
2832 // instructions in the bundle or reach the end of the block. If Front is the
2833 // last instruction in program order, LastInst will be set to Front, and we
2834 // will visit all the remaining instructions in the block.
2835 //
2836 // One of the reasons we exit early from buildTree_rec is to place an upper
2837 // bound on compile-time. Thus, taking an additional compile-time hit here is
2838 // not ideal. However, this should be exceedingly rare since it requires that
2839 // we both exit early from buildTree_rec and that the bundle be out-of-order
2840 // (causing us to iterate all the way to the end of the block).
2841 if (!LastInst) {
13
Taking true branch
2842 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2843 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2844 if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2845 LastInst = &I;
2846 if (Bundle.empty())
2847 break;
2848 }
2849 }
2850
2851 // Set the insertion point after the last instruction in the bundle. Set the
2852 // debug location to Front.
2853 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
14
Called C++ object pointer is null
2854 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2855}
2856
2857Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2858 Value *Vec = UndefValue::get(Ty);
2859 // Generate the 'InsertElement' instruction.
2860 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2861 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2862 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2863 GatherSeq.insert(Insrt);
2864 CSEBlocks.insert(Insrt->getParent());
2865
2866 // Add to our 'need-to-extract' list.
2867 if (TreeEntry *E = getTreeEntry(VL[i])) {
2868 // Find which lane we need to extract.
2869 int FoundLane = -1;
2870 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2871 // Is this the lane of the scalar that we are looking for ?
2872 if (E->Scalars[Lane] == VL[i]) {
2873 FoundLane = Lane;
2874 break;
2875 }
2876 }
2877 assert(FoundLane >= 0 && "Could not find the correct lane")((FoundLane >= 0 && "Could not find the correct lane"
) ? static_cast<void> (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2877, __PRETTY_FUNCTION__))
;
2878 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2879 }
2880 }
2881 }
2882
2883 return Vec;
2884}
2885
2886Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const {
2887 if (const TreeEntry *En = getTreeEntry(OpValue)) {
2888 if (En->isSame(VL) && En->VectorizedValue)
2889 return En->VectorizedValue;
2890 }
2891 return nullptr;
2892}
2893
2894Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2895 InstructionsState S = getSameOpcode(VL);
2896 if (S.Opcode) {
2897 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2898 if (E->isSame(VL))
2899 return vectorizeTree(E);
2900 }
2901 }
2902
2903 Type *ScalarTy = S.OpValue->getType();
2904 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2905 ScalarTy = SI->getValueOperand()->getType();
2906 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2907
2908 return Gather(VL, VecTy);
2909}
2910
2911Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2912 IRBuilder<>::InsertPointGuard Guard(Builder);
2913
2914 if (E->VectorizedValue) {
1
Assuming the condition is false
2
Taking false branch
2915 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->State.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->State.OpValue << ".\n"; } } while (false)
;
2916 return E->VectorizedValue;
2917 }
2918
2919 Instruction *VL0 = cast<Instruction>(E->State.OpValue);
2920 Type *ScalarTy = VL0->getType();
2921 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3
Taking false branch
2922 ScalarTy = SI->getValueOperand()->getType();
2923 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2924
2925 if (E->NeedToGather) {
4
Assuming the condition is false
5
Taking false branch
2926 setInsertPointAfterBundle(E->Scalars, VL0);
2927 auto *V = Gather(E->Scalars, VecTy);
2928 E->VectorizedValue = V;
2929 return V;
2930 }
2931
2932 unsigned ShuffleOrOp = E->State.IsAltShuffle ?
6
Assuming the condition is false
7
'?' condition is false
2933 (unsigned) Instruction::ShuffleVector : E->State.Opcode;
2934 switch (ShuffleOrOp) {
8
Control jumps to 'case Call:' at line 3218
2935 case Instruction::PHI: {
2936 PHINode *PH = dyn_cast<PHINode>(VL0);
2937 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2938 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2939 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2940 E->VectorizedValue = NewPhi;
2941
2942 // PHINodes may have multiple entries from the same block. We want to
2943 // visit every block once.
2944 SmallSet<BasicBlock*, 4> VisitedBBs;
2945
2946 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2947 ValueList Operands;
2948 BasicBlock *IBB = PH->getIncomingBlock(i);
2949
2950 if (!VisitedBBs.insert(IBB).second) {
2951 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2952 continue;
2953 }
2954
2955 // Prepare the operand vector.
2956 for (Value *V : E->Scalars)
2957 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2958
2959 Builder.SetInsertPoint(IBB->getTerminator());
2960 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2961 Value *Vec = vectorizeTree(Operands);
2962 NewPhi->addIncoming(Vec, IBB);
2963 }
2964
2965 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2966, __PRETTY_FUNCTION__))
2966 "Invalid number of incoming values")((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2966, __PRETTY_FUNCTION__))
;
2967 return NewPhi;
2968 }
2969
2970 case Instruction::ExtractElement: {
2971 if (canReuseExtract(E->Scalars, VL0)) {
2972 Value *V = VL0->getOperand(0);
2973 E->VectorizedValue = V;
2974 return V;
2975 }
2976 setInsertPointAfterBundle(E->Scalars, VL0);
2977 auto *V = Gather(E->Scalars, VecTy);
2978 E->VectorizedValue = V;
2979 return V;
2980 }
2981 case Instruction::ExtractValue: {
2982 if (canReuseExtract(E->Scalars, VL0)) {
2983 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2984 Builder.SetInsertPoint(LI);
2985 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2986 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2987 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2988 E->VectorizedValue = V;
2989 return propagateMetadata(V, E->Scalars);
2990 }
2991 setInsertPointAfterBundle(E->Scalars, VL0);
2992 auto *V = Gather(E->Scalars, VecTy);
2993 E->VectorizedValue = V;
2994 return V;
2995 }
2996 case Instruction::ZExt:
2997 case Instruction::SExt:
2998 case Instruction::FPToUI:
2999 case Instruction::FPToSI:
3000 case Instruction::FPExt:
3001 case Instruction::PtrToInt:
3002 case Instruction::IntToPtr:
3003 case Instruction::SIToFP:
3004 case Instruction::UIToFP:
3005 case Instruction::Trunc:
3006 case Instruction::FPTrunc:
3007 case Instruction::BitCast: {
3008 ValueList INVL;
3009 for (Value *V : E->Scalars)
3010 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3011
3012 setInsertPointAfterBundle(E->Scalars, VL0);
3013
3014 Value *InVec = vectorizeTree(INVL);
3015
3016 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3017 return V;
3018
3019 CastInst *CI = dyn_cast<CastInst>(VL0);
3020 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3021 E->VectorizedValue = V;
3022 ++NumVectorInstructions;
3023 return V;
3024 }
3025 case Instruction::FCmp:
3026 case Instruction::ICmp: {
3027 ValueList LHSV, RHSV;
3028 for (Value *V : E->Scalars) {
3029 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3030 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3031 }
3032
3033 setInsertPointAfterBundle(E->Scalars, VL0);
3034
3035 Value *L = vectorizeTree(LHSV);
3036 Value *R = vectorizeTree(RHSV);
3037
3038 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3039 return V;
3040
3041 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3042 Value *V;
3043 if (E->State.Opcode == Instruction::FCmp)
3044 V = Builder.CreateFCmp(P0, L, R);
3045 else
3046 V = Builder.CreateICmp(P0, L, R);
3047
3048 E->VectorizedValue = V;
3049 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3050 ++NumVectorInstructions;
3051 return V;
3052 }
3053 case Instruction::Select: {
3054 ValueList TrueVec, FalseVec, CondVec;
3055 for (Value *V : E->Scalars) {
3056 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3057 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3058 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3059 }
3060
3061 setInsertPointAfterBundle(E->Scalars, VL0);
3062
3063 Value *Cond = vectorizeTree(CondVec);
3064 Value *True = vectorizeTree(TrueVec);
3065 Value *False = vectorizeTree(FalseVec);
3066
3067 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3068 return V;
3069
3070 Value *V = Builder.CreateSelect(Cond, True, False);
3071 E->VectorizedValue = V;
3072 ++NumVectorInstructions;
3073 return V;
3074 }
3075 case Instruction::Add:
3076 case Instruction::FAdd:
3077 case Instruction::Sub:
3078 case Instruction::FSub:
3079 case Instruction::Mul:
3080 case Instruction::FMul:
3081 case Instruction::UDiv:
3082 case Instruction::SDiv:
3083 case Instruction::FDiv:
3084 case Instruction::URem:
3085 case Instruction::SRem:
3086 case Instruction::FRem:
3087 case Instruction::Shl:
3088 case Instruction::LShr:
3089 case Instruction::AShr:
3090 case Instruction::And:
3091 case Instruction::Or:
3092 case Instruction::Xor: {
3093 ValueList LHSVL, RHSVL;
3094 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3095 reorderInputsAccordingToOpcode(E->State.Opcode, E->Scalars, LHSVL,
3096 RHSVL);
3097 else
3098 for (Value *V : E->Scalars) {
3099 auto *I = cast<Instruction>(V);
3100 if (I->getOpcode() == E->State.Opcode) {
3101 LHSVL.push_back(I->getOperand(0));
3102 RHSVL.push_back(I->getOperand(1));
3103 } else {
3104 LHSVL.push_back(V);
3105 RHSVL.push_back(
3106 getDefaultConstantForOpcode(E->State.Opcode, I->getType()));
3107 }
3108 }
3109
3110 setInsertPointAfterBundle(E->Scalars, VL0);
3111
3112 Value *LHS = vectorizeTree(LHSVL);
3113 Value *RHS = vectorizeTree(RHSVL);
3114
3115 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3116 return V;
3117
3118 Value *V = Builder.CreateBinOp(
3119 static_cast<Instruction::BinaryOps>(E->State.Opcode), LHS, RHS);
3120 E->VectorizedValue = V;
3121 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3122 ++NumVectorInstructions;
3123
3124 if (Instruction *I = dyn_cast<Instruction>(V))
3125 return propagateMetadata(I, E->Scalars);
3126
3127 return V;
3128 }
3129 case Instruction::Load: {
3130 // Loads are inserted at the head of the tree because we don't want to
3131 // sink them all the way down past store instructions.
3132 setInsertPointAfterBundle(E->Scalars, VL0);
3133
3134 LoadInst *LI = cast<LoadInst>(VL0);
3135 Type *ScalarLoadTy = LI->getType();
3136 unsigned AS = LI->getPointerAddressSpace();
3137
3138 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3139 VecTy->getPointerTo(AS));
3140
3141 // The pointer operand uses an in-tree scalar so we add the new BitCast to
3142 // ExternalUses list to make sure that an extract will be generated in the
3143 // future.
3144 Value *PO = LI->getPointerOperand();
3145 if (getTreeEntry(PO))
3146 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3147
3148 unsigned Alignment = LI->getAlignment();
3149 LI = Builder.CreateLoad(VecPtr);
3150 if (!Alignment) {
3151 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3152 }
3153 LI->setAlignment(Alignment);
3154 E->VectorizedValue = LI;
3155 ++NumVectorInstructions;
3156 return propagateMetadata(LI, E->Scalars);
3157 }
3158 case Instruction::Store: {
3159 StoreInst *SI = cast<StoreInst>(VL0);
3160 unsigned Alignment = SI->getAlignment();
3161 unsigned AS = SI->getPointerAddressSpace();
3162
3163 ValueList ScalarStoreValues;
3164 for (Value *V : E->Scalars)
3165 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3166
3167 setInsertPointAfterBundle(E->Scalars, VL0);
3168
3169 Value *VecValue = vectorizeTree(ScalarStoreValues);
3170 Value *ScalarPtr = SI->getPointerOperand();
3171 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3172 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3173
3174 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3175 // ExternalUses to make sure that an extract will be generated in the
3176 // future.
3177 if (getTreeEntry(ScalarPtr))
3178 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3179
3180 if (!Alignment)
3181 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3182
3183 S->setAlignment(Alignment);
3184 E->VectorizedValue = S;
3185 ++NumVectorInstructions;
3186 return propagateMetadata(S, E->Scalars);
3187 }
3188 case Instruction::GetElementPtr: {
3189 setInsertPointAfterBundle(E->Scalars, VL0);
3190
3191 ValueList Op0VL;
3192 for (Value *V : E->Scalars)
3193 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3194
3195 Value *Op0 = vectorizeTree(Op0VL);
3196
3197 std::vector<Value *> OpVecs;
3198 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3199 ++j) {
3200 ValueList OpVL;
3201 for (Value *V : E->Scalars)
3202 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3203
3204 Value *OpVec = vectorizeTree(OpVL);
3205 OpVecs.push_back(OpVec);
3206 }
3207
3208 Value *V = Builder.CreateGEP(
3209 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3210 E->VectorizedValue = V;
3211 ++NumVectorInstructions;
3212
3213 if (Instruction *I = dyn_cast<Instruction>(V))
3214 return propagateMetadata(I, E->Scalars);
3215
3216 return V;
3217 }
3218 case Instruction::Call: {
3219 CallInst *CI = cast<CallInst>(VL0);
3220 setInsertPointAfterBundle(E->Scalars, VL0);
9
Calling 'BoUpSLP::setInsertPointAfterBundle'
3221 Function *FI;
3222 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3223 Value *ScalarArg = nullptr;
3224 if (CI && (FI = CI->getCalledFunction())) {
3225 IID = FI->getIntrinsicID();
3226 }
3227 std::vector<Value *> OpVecs;
3228 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3229 ValueList OpVL;
3230 // ctlz,cttz and powi are special intrinsics whose second argument is
3231 // a scalar. This argument should not be vectorized.
3232 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3233 CallInst *CEI = cast<CallInst>(VL0);
3234 ScalarArg = CEI->getArgOperand(j);
3235 OpVecs.push_back(CEI->getArgOperand(j));
3236 continue;
3237 }
3238 for (Value *V : E->Scalars) {
3239 CallInst *CEI = cast<CallInst>(V);
3240 OpVL.push_back(CEI->getArgOperand(j));
3241 }
3242
3243 Value *OpVec = vectorizeTree(OpVL);
3244 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
3245 OpVecs.push_back(OpVec);
3246 }
3247
3248 Module *M = F->getParent();
3249 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3250 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3251 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3252 SmallVector<OperandBundleDef, 1> OpBundles;
3253 CI->getOperandBundlesAsDefs(OpBundles);
3254 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3255
3256 // The scalar argument uses an in-tree scalar so we add the new vectorized
3257 // call to ExternalUses list to make sure that an extract will be
3258 // generated in the future.
3259 if (ScalarArg && getTreeEntry(ScalarArg))
3260 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3261
3262 E->VectorizedValue = V;
3263 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3264 ++NumVectorInstructions;
3265 return V;
3266 }
3267 case Instruction::ShuffleVector: {
3268 ValueList LHSVL, RHSVL;
3269 assert(Instruction::isBinaryOp(E->State.Opcode) &&((Instruction::isBinaryOp(E->State.Opcode) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("Instruction::isBinaryOp(E->State.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3270, __PRETTY_FUNCTION__))
3270 "Invalid Shuffle Vector Operand")((Instruction::isBinaryOp(E->State.Opcode) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("Instruction::isBinaryOp(E->State.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3270, __PRETTY_FUNCTION__))
;
3271 reorderAltShuffleOperands(E->State.Opcode, E->Scalars, LHSVL, RHSVL);
3272 setInsertPointAfterBundle(E->Scalars, VL0);
3273
3274 Value *LHS = vectorizeTree(LHSVL);
3275 Value *RHS = vectorizeTree(RHSVL);
3276
3277 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3278 return V;
3279
3280 // Create a vector of LHS op1 RHS
3281 Value *V0 = Builder.CreateBinOp(
3282 static_cast<Instruction::BinaryOps>(E->State.Opcode), LHS, RHS);
3283
3284 unsigned AltOpcode = getAltOpcode(E->State.Opcode);
3285 // Create a vector of LHS op2 RHS
3286 Value *V1 = Builder.CreateBinOp(
3287 static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3288
3289 // Create shuffle to take alternate operations from the vector.
3290 // Also, gather up odd and even scalar ops to propagate IR flags to
3291 // each vector operation.
3292 ValueList OddScalars, EvenScalars;
3293 unsigned e = E->Scalars.size();
3294 SmallVector<Constant *, 8> Mask(e);
3295 for (unsigned i = 0; i < e; ++i) {
3296 if (isOdd(i)) {
3297 Mask[i] = Builder.getInt32(e + i);
3298 OddScalars.push_back(E->Scalars[i]);
3299 } else {
3300 Mask[i] = Builder.getInt32(i);
3301 EvenScalars.push_back(E->Scalars[i]);
3302 }
3303 }
3304
3305 Value *ShuffleMask = ConstantVector::get(Mask);
3306 InstructionsState S = getSameOpcode(EvenScalars);
3307 assert(!S.IsAltShuffle && "Unexpected alternate opcode")((!S.IsAltShuffle && "Unexpected alternate opcode") ?
static_cast<void> (0) : __assert_fail ("!S.IsAltShuffle && \"Unexpected alternate opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3307, __PRETTY_FUNCTION__))
;
3308 propagateIRFlags(V0, EvenScalars, S.OpValue);
3309
3310 S = getSameOpcode(OddScalars);
3311 assert(!S.IsAltShuffle && "Unexpected alternate opcode")((!S.IsAltShuffle && "Unexpected alternate opcode") ?
static_cast<void> (0) : __assert_fail ("!S.IsAltShuffle && \"Unexpected alternate opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3311, __PRETTY_FUNCTION__))
;
3312 propagateIRFlags(V1, OddScalars, S.OpValue);
3313
3314 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3315 E->VectorizedValue = V;
3316 ++NumVectorInstructions;
3317 if (Instruction *I = dyn_cast<Instruction>(V))
3318 return propagateMetadata(I, E->Scalars);
3319
3320 return V;
3321 }
3322 default:
3323 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3323)
;
3324 }
3325 return nullptr;
3326}
3327
3328Value *BoUpSLP::vectorizeTree() {
3329 ExtraValueToDebugLocsMap ExternallyUsedValues;
3330 return vectorizeTree(ExternallyUsedValues);
3331}
3332
3333Value *
3334BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3335 // All blocks must be scheduled before any instructions are inserted.
3336 for (auto &BSIter : BlocksSchedules) {
3337 scheduleBlock(BSIter.second.get());
3338 }
3339
3340 Builder.SetInsertPoint(&F->getEntryBlock().front());
3341 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3342
3343 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3344 // vectorized root. InstCombine will then rewrite the entire expression. We
3345 // sign extend the extracted values below.
3346 auto *ScalarRoot = VectorizableTree[0].State.OpValue;
3347 if (MinBWs.count(ScalarRoot)) {
3348 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3349 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3350 auto BundleWidth = VectorizableTree[0].Scalars.size();
3351 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3352 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3353 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3354 VectorizableTree[0].VectorizedValue = Trunc;
3355 }
3356
3357 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
3358
3359 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3360 // specified by ScalarType.
3361 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3362 if (!MinBWs.count(ScalarRoot))
3363 return Ex;
3364 if (MinBWs[ScalarRoot].second)
3365 return Builder.CreateSExt(Ex, ScalarType);
3366 return Builder.CreateZExt(Ex, ScalarType);
3367 };
3368
3369 // Extract all of the elements with the external uses.
3370 for (const auto &ExternalUse : ExternalUses) {
3371 Value *Scalar = ExternalUse.Scalar;
3372 llvm::User *User = ExternalUse.User;
3373
3374 // Skip users that we already RAUW. This happens when one instruction
3375 // has multiple uses of the same value.
3376 if (User && !is_contained(Scalar->users(), User))
3377 continue;
3378 TreeEntry *E = getTreeEntry(Scalar);
3379 assert(E && "Invalid scalar")((E && "Invalid scalar") ? static_cast<void> (0
) : __assert_fail ("E && \"Invalid scalar\"", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3379, __PRETTY_FUNCTION__))
;
3380 assert(!E->NeedToGather && "Extracting from a gather list")((!E->NeedToGather && "Extracting from a gather list"
) ? static_cast<void> (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3380, __PRETTY_FUNCTION__))
;
3381
3382 Value *Vec = E->VectorizedValue;
3383 assert(Vec && "Can't find vectorizable value")((Vec && "Can't find vectorizable value") ? static_cast
<void> (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3383, __PRETTY_FUNCTION__))
;
3384
3385 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3386 // If User == nullptr, the Scalar is used as extra arg. Generate
3387 // ExtractElement instruction and update the record for this scalar in
3388 // ExternallyUsedValues.
3389 if (!User) {
3390 assert(ExternallyUsedValues.count(Scalar) &&((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3392, __PRETTY_FUNCTION__))
3391 "Scalar with nullptr as an external user must be registered in "((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3392, __PRETTY_FUNCTION__))
3392 "ExternallyUsedValues map")((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3392, __PRETTY_FUNCTION__))
;
3393 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3394 Builder.SetInsertPoint(VecI->getParent(),
3395 std::next(VecI->getIterator()));
3396 } else {
3397 Builder.SetInsertPoint(&F->getEntryBlock().front());
3398 }
3399 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3400 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3401 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3402 auto &Locs = ExternallyUsedValues[Scalar];
3403 ExternallyUsedValues.insert({Ex, Locs});
3404 ExternallyUsedValues.erase(Scalar);
3405 continue;
3406 }
3407
3408 // Generate extracts for out-of-tree users.
3409 // Find the insertion point for the extractelement lane.
3410 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3411 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3412 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3413 if (PH->getIncomingValue(i) == Scalar) {
3414 TerminatorInst *IncomingTerminator =
3415 PH->getIncomingBlock(i)->getTerminator();
3416 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3417 Builder.SetInsertPoint(VecI->getParent(),
3418 std::next(VecI->getIterator()));
3419 } else {
3420 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3421 }
3422 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3423 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3424 CSEBlocks.insert(PH->getIncomingBlock(i));
3425 PH->setOperand(i, Ex);
3426 }
3427 }
3428 } else {
3429 Builder.SetInsertPoint(cast<Instruction>(User));
3430 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3431 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3432 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3433 User->replaceUsesOfWith(Scalar, Ex);
3434 }
3435 } else {
3436 Builder.SetInsertPoint(&F->getEntryBlock().front());
3437 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3438 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3439 CSEBlocks.insert(&F->getEntryBlock());
3440 User->replaceUsesOfWith(Scalar, Ex);
3441 }
3442
3443 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
3444 }
3445
3446 // For each vectorized value:
3447 for (TreeEntry &EIdx : VectorizableTree) {
3448 TreeEntry *Entry = &EIdx;
3449
3450 // No need to handle users of gathered values.
3451 if (Entry->NeedToGather)
3452 continue;
3453
3454 assert(Entry->VectorizedValue && "Can't find vectorizable value")((Entry->VectorizedValue && "Can't find vectorizable value"
) ? static_cast<void> (0) : __assert_fail ("Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3454, __PRETTY_FUNCTION__))
;
3455
3456 // For each lane:
3457 const unsigned Opcode = Entry->State.Opcode;
3458 const unsigned AltOpcode = getAltOpcode(Opcode);
3459 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3460 Value *Scalar = Entry->Scalars[Lane];
3461
3462 if (!sameOpcodeOrAlt(Opcode, AltOpcode,
3463 cast<Instruction>(Scalar)->getOpcode()))
3464 continue;
3465
3466 Type *Ty = Scalar->getType();
3467 if (!Ty->isVoidTy()) {
3468#ifndef NDEBUG
3469 for (User *U : Scalar->users()) {
3470 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
3471
3472 // It is legal to replace users in the ignorelist by undef.
3473 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&(((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
"Replacing out-of-tree value with undef") ? static_cast<void
> (0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3474, __PRETTY_FUNCTION__))
3474 "Replacing out-of-tree value with undef")(((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
"Replacing out-of-tree value with undef") ? static_cast<void
> (0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3474, __PRETTY_FUNCTION__))
;
3475 }
3476#endif
3477 Value *Undef = UndefValue::get(Ty);
3478 Scalar->replaceAllUsesWith(Undef);
3479 }
3480 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
3481 eraseInstruction(cast<Instruction>(Scalar));
3482 }
3483 }
3484
3485 Builder.ClearInsertionPoint();
3486
3487 return VectorizableTree[0].VectorizedValue;
3488}
3489
3490void BoUpSLP::optimizeGatherSequence() {
3491 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
3492 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
3493 // LICM InsertElementInst sequences.
3494 for (Instruction *it : GatherSeq) {
3495 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
3496
3497 if (!Insert)
3498 continue;
3499
3500 // Check if this block is inside a loop.
3501 Loop *L = LI->getLoopFor(Insert->getParent());
3502 if (!L)
3503 continue;
3504
3505 // Check if it has a preheader.
3506 BasicBlock *PreHeader = L->getLoopPreheader();
3507 if (!PreHeader)
3508 continue;
3509
3510 // If the vector or the element that we insert into it are
3511 // instructions that are defined in this basic block then we can't
3512 // hoist this instruction.
3513 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
3514 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
3515 if (CurrVec && L->contains(CurrVec))
3516 continue;
3517 if (NewElem && L->contains(NewElem))
3518 continue;
3519
3520 // We can hoist this instruction. Move it to the pre-header.
3521 Insert->moveBefore(PreHeader->getTerminator());
3522 }
3523
3524 // Make a list of all reachable blocks in our CSE queue.
3525 SmallVector<const DomTreeNode *, 8> CSEWorkList;
3526 CSEWorkList.reserve(CSEBlocks.size());
3527 for (BasicBlock *BB : CSEBlocks)
3528 if (DomTreeNode *N = DT->getNode(BB)) {
3529 assert(DT->isReachableFromEntry(N))((DT->isReachableFromEntry(N)) ? static_cast<void> (
0) : __assert_fail ("DT->isReachableFromEntry(N)", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3529, __PRETTY_FUNCTION__))
;
3530 CSEWorkList.push_back(N);
3531 }
3532
3533 // Sort blocks by domination. This ensures we visit a block after all blocks
3534 // dominating it are visited.
3535 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3536 [this](const DomTreeNode *A, const DomTreeNode *B) {
3537 return DT->properlyDominates(A, B);
3538 });
3539
3540 // Perform O(N^2) search over the gather sequences and merge identical
3541 // instructions. TODO: We can further optimize this scan if we split the
3542 // instructions into different buckets based on the insert lane.
3543 SmallVector<Instruction *, 16> Visited;
3544 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3545 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3546, __PRETTY_FUNCTION__))
3546 "Worklist not sorted properly!")(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3546, __PRETTY_FUNCTION__))
;
3547 BasicBlock *BB = (*I)->getBlock();
3548 // For all instructions in blocks containing gather sequences:
3549 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3550 Instruction *In = &*it++;
3551 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3552 continue;
3553
3554 // Check if we can replace this instruction with any of the
3555 // visited instructions.
3556 for (Instruction *v : Visited) {
3557 if (In->isIdenticalTo(v) &&
3558 DT->dominates(v->getParent(), In->getParent())) {
3559 In->replaceAllUsesWith(v);
3560 eraseInstruction(In);
3561 In = nullptr;
3562 break;
3563 }
3564 }
3565 if (In) {
3566 assert(!is_contained(Visited, In))((!is_contained(Visited, In)) ? static_cast<void> (0) :
__assert_fail ("!is_contained(Visited, In)", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3566, __PRETTY_FUNCTION__))
;
3567 Visited.push_back(In);
3568 }
3569 }
3570 }
3571 CSEBlocks.clear();
3572 GatherSeq.clear();
3573}
3574
3575// Groups the instructions to a bundle (which is then a single scheduling entity)
3576// and schedules instructions until the bundle gets ready.
3577bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3578 BoUpSLP *SLP, Value *OpValue) {
3579 if (isa<PHINode>(OpValue))
3580 return true;
3581
3582 // Initialize the instruction bundle.
3583 Instruction *OldScheduleEnd = ScheduleEnd;
3584 ScheduleData *PrevInBundle = nullptr;
3585 ScheduleData *Bundle = nullptr;
3586 bool ReSchedule = false;
3587 DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *OpValue
<< "\n"; } } while (false)
;
3588
3589 // Make sure that the scheduling region contains all
3590 // instructions of the bundle.
3591 for (Value *V : VL) {
3592 if (!extendSchedulingRegion(V, OpValue))
3593 return false;
3594 }
3595
3596 for (Value *V : VL) {
3597 ScheduleData *BundleMember = getScheduleData(V, isOneOf(OpValue, V));
3598 assert(BundleMember &&((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3599, __PRETTY_FUNCTION__))
3599 "no ScheduleData for bundle member (maybe not in same basic block)")((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3599, __PRETTY_FUNCTION__))
;
3600 if (BundleMember->IsScheduled) {
3601 // A bundle member was scheduled as single instruction before and now
3602 // needs to be scheduled as part of the bundle. We just get rid of the
3603 // existing schedule.
3604 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3605 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
3606 ReSchedule = true;
3607 }
3608 assert(BundleMember->isSchedulingEntity() &&((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3609, __PRETTY_FUNCTION__))
3609 "bundle member already part of other bundle")((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3609, __PRETTY_FUNCTION__))
;
3610 if (PrevInBundle) {
3611 PrevInBundle->NextInBundle = BundleMember;
3612 } else {
3613 Bundle = BundleMember;
3614 }
3615 BundleMember->UnscheduledDepsInBundle = 0;
3616 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3617
3618 // Group the instructions to a bundle.
3619 BundleMember->FirstInBundle = Bundle;
3620 PrevInBundle = BundleMember;
3621 }
3622 if (ScheduleEnd != OldScheduleEnd) {
3623 // The scheduling region got new instructions at the lower end (or it is a
3624 // new region for the first bundle). This makes it necessary to
3625 // recalculate all dependencies.
3626 // It is seldom that this needs to be done a second time after adding the
3627 // initial bundle to the region.
3628 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3629 doForAllOpcodes(I, [](ScheduleData *SD) {
3630 SD->clearDependencies();
3631 });
3632 }
3633 ReSchedule = true;
3634 }
3635 if (ReSchedule) {
3636 resetSchedule();
3637 initialFillReadyList(ReadyInsts);
3638 }
3639
3640 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
3641 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
3642
3643 calculateDependencies(Bundle, true, SLP);
3644
3645 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3646 // means that there are no cyclic dependencies and we can schedule it.
3647 // Note that's important that we don't "schedule" the bundle yet (see
3648 // cancelScheduling).
3649 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3650
3651 ScheduleData *pickedSD = ReadyInsts.back();
3652 ReadyInsts.pop_back();
3653
3654 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3655 schedule(pickedSD, ReadyInsts);
3656 }
3657 }
3658 if (!Bundle->isReady()) {
3659 cancelScheduling(VL, OpValue);
3660 return false;
3661 }
3662 return true;
3663}
3664
3665void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3666 Value *OpValue) {
3667 if (isa<PHINode>(OpValue))
3668 return;
3669
3670 ScheduleData *Bundle = getScheduleData(OpValue)->FirstInBundle;
3671 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3672 assert(!Bundle->IsScheduled &&((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3673, __PRETTY_FUNCTION__))
3673 "Can't cancel bundle which is already scheduled")((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3673, __PRETTY_FUNCTION__))
;
3674 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3675, __PRETTY_FUNCTION__))
3675 "tried to unbundle something which is not a bundle")((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3675, __PRETTY_FUNCTION__))
;
3676
3677 // Un-bundle: make single instructions out of the bundle.
3678 ScheduleData *BundleMember = Bundle;
3679 while (BundleMember) {
3680 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")((BundleMember->FirstInBundle == Bundle && "corrupt bundle links"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3680, __PRETTY_FUNCTION__))
;
3681 BundleMember->FirstInBundle = BundleMember;
3682 ScheduleData *Next = BundleMember->NextInBundle;
3683 BundleMember->NextInBundle = nullptr;
3684 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3685 if (BundleMember->UnscheduledDepsInBundle == 0) {
3686 ReadyInsts.insert(BundleMember);
3687 }
3688 BundleMember = Next;
3689 }
3690}
3691
3692BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3693 // Allocate a new ScheduleData for the instruction.
3694 if (ChunkPos >= ChunkSize) {
3695 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3696 ChunkPos = 0;
3697 }
3698 return &(ScheduleDataChunks.back()[ChunkPos++]);
3699}
3700
3701bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3702 Value *OpValue) {
3703 if (getScheduleData(V, isOneOf(OpValue, V)))
3704 return true;
3705 Instruction *I = dyn_cast<Instruction>(V);
3706 assert(I && "bundle member must be an instruction")((I && "bundle member must be an instruction") ? static_cast
<void> (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3706, __PRETTY_FUNCTION__))
;
3707 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")((!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3707, __PRETTY_FUNCTION__))
;
3708 auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3709 ScheduleData *ISD = getScheduleData(I);
3710 if (!ISD)
3711 return false;
3712 assert(isInSchedulingRegion(ISD) &&((isInSchedulingRegion(ISD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3713, __PRETTY_FUNCTION__))
3713 "ScheduleData not in scheduling region")((isInSchedulingRegion(ISD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3713, __PRETTY_FUNCTION__))
;
3714 ScheduleData *SD = allocateScheduleDataChunks();
3715 SD->Inst = I;
3716 SD->init(SchedulingRegionID, OpValue);
3717 ExtraScheduleDataMap[I][OpValue] = SD;
3718 return true;
3719 };
3720 if (CheckSheduleForI(I))
3721 return true;
3722 if (!ScheduleStart) {
3723 // It's the first instruction in the new region.
3724 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3725 ScheduleStart = I;
3726 ScheduleEnd = I->getNextNode();
3727 if (isOneOf(OpValue, I) != I)
3728 CheckSheduleForI(I);
3729 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3729, __PRETTY_FUNCTION__))
;
3730 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3731 return true;
3732 }
3733 // Search up and down at the same time, because we don't know if the new
3734 // instruction is above or below the existing scheduling region.
3735 BasicBlock::reverse_iterator UpIter =
3736 ++ScheduleStart->getIterator().getReverse();
3737 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3738 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3739 BasicBlock::iterator LowerEnd = BB->end();
3740 while (true) {
3741 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3742 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3743 return false;
3744 }
3745
3746 if (UpIter != UpperEnd) {
3747 if (&*UpIter == I) {
3748 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3749 ScheduleStart = I;
3750 if (isOneOf(OpValue, I) != I)
3751 CheckSheduleForI(I);
3752 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3753 return true;
3754 }
3755 UpIter++;
3756 }
3757 if (DownIter != LowerEnd) {
3758 if (&*DownIter == I) {
3759 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3760 nullptr);
3761 ScheduleEnd = I->getNextNode();
3762 if (isOneOf(OpValue, I) != I)
3763 CheckSheduleForI(I);
3764 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3764, __PRETTY_FUNCTION__))
;
3765 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
3766 return true;
3767 }
3768 DownIter++;
3769 }
3770 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3771, __PRETTY_FUNCTION__))
3771 "instruction not found in block")(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3771, __PRETTY_FUNCTION__))
;
3772 }
3773 return true;
3774}
3775
3776void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3777 Instruction *ToI,
3778 ScheduleData *PrevLoadStore,
3779 ScheduleData *NextLoadStore) {
3780 ScheduleData *CurrentLoadStore = PrevLoadStore;
3781 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3782 ScheduleData *SD = ScheduleDataMap[I];
3783 if (!SD) {
3784 SD = allocateScheduleDataChunks();
3785 ScheduleDataMap[I] = SD;
3786 SD->Inst = I;
3787 }
3788 assert(!isInSchedulingRegion(SD) &&((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3789, __PRETTY_FUNCTION__))
3789 "new ScheduleData already in scheduling region")((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3789, __PRETTY_FUNCTION__))
;
3790 SD->init(SchedulingRegionID, I);
3791
3792 if (I->mayReadOrWriteMemory() &&
3793 (!isa<IntrinsicInst>(I) ||
3794 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
3795 // Update the linked list of memory accessing instructions.
3796 if (CurrentLoadStore) {
3797 CurrentLoadStore->NextLoadStore = SD;
3798 } else {
3799 FirstLoadStoreInRegion = SD;
3800 }
3801 CurrentLoadStore = SD;
3802 }
3803 }
3804 if (NextLoadStore) {
3805 if (CurrentLoadStore)
3806 CurrentLoadStore->NextLoadStore = NextLoadStore;
3807 } else {
3808 LastLoadStoreInRegion = CurrentLoadStore;
3809 }
3810}
3811
3812void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3813 bool InsertInReadyList,
3814 BoUpSLP *SLP) {
3815 assert(SD->isSchedulingEntity())((SD->isSchedulingEntity()) ? static_cast<void> (0) :
__assert_fail ("SD->isSchedulingEntity()", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3815, __PRETTY_FUNCTION__))
;
3816
3817 SmallVector<ScheduleData *, 10> WorkList;
3818 WorkList.push_back(SD);
3819
3820 while (!WorkList.empty()) {
3821 ScheduleData *SD = WorkList.back();
3822 WorkList.pop_back();
3823
3824 ScheduleData *BundleMember = SD;
3825 while (BundleMember) {
3826 assert(isInSchedulingRegion(BundleMember))((isInSchedulingRegion(BundleMember)) ? static_cast<void>
(0) : __assert_fail ("isInSchedulingRegion(BundleMember)", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3826, __PRETTY_FUNCTION__))
;
3827 if (!BundleMember->hasValidDependencies()) {
3828
3829 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
3830 BundleMember->Dependencies = 0;
3831 BundleMember->resetUnscheduledDeps();
3832
3833 // Handle def-use chain dependencies.
3834 if (BundleMember->OpValue != BundleMember->Inst) {
3835 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
3836 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3837 BundleMember->Dependencies++;
3838 ScheduleData *DestBundle = UseSD->FirstInBundle;
3839 if (!DestBundle->IsScheduled)
3840 BundleMember->incrementUnscheduledDeps(1);
3841 if (!DestBundle->hasValidDependencies())
3842 WorkList.push_back(DestBundle);
3843 }
3844 } else {
3845 for (User *U : BundleMember->Inst->users()) {
3846 if (isa<Instruction>(U)) {
3847 ScheduleData *UseSD = getScheduleData(U);
3848 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3849 BundleMember->Dependencies++;
3850 ScheduleData *DestBundle = UseSD->FirstInBundle;
3851 if (!DestBundle->IsScheduled)
3852 BundleMember->incrementUnscheduledDeps(1);
3853 if (!DestBundle->hasValidDependencies())
3854 WorkList.push_back(DestBundle);
3855 }
3856 } else {
3857 // I'm not sure if this can ever happen. But we need to be safe.
3858 // This lets the instruction/bundle never be scheduled and
3859 // eventually disable vectorization.
3860 BundleMember->Dependencies++;
3861 BundleMember->incrementUnscheduledDeps(1);
3862 }
3863 }
3864 }
3865
3866 // Handle the memory dependencies.
3867 ScheduleData *DepDest = BundleMember->NextLoadStore;
3868 if (DepDest) {
3869 Instruction *SrcInst = BundleMember->Inst;
3870 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3871 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3872 unsigned numAliased = 0;
3873 unsigned DistToSrc = 1;
3874
3875 while (DepDest) {
3876 assert(isInSchedulingRegion(DepDest))((isInSchedulingRegion(DepDest)) ? static_cast<void> (0
) : __assert_fail ("isInSchedulingRegion(DepDest)", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3876, __PRETTY_FUNCTION__))
;
3877
3878 // We have two limits to reduce the complexity:
3879 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3880 // SLP->isAliased (which is the expensive part in this loop).
3881 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3882 // the whole loop (even if the loop is fast, it's quadratic).
3883 // It's important for the loop break condition (see below) to
3884 // check this limit even between two read-only instructions.
3885 if (DistToSrc >= MaxMemDepDistance ||
3886 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3887 (numAliased >= AliasedCheckLimit ||
3888 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3889
3890 // We increment the counter only if the locations are aliased
3891 // (instead of counting all alias checks). This gives a better
3892 // balance between reduced runtime and accurate dependencies.
3893 numAliased++;
3894
3895 DepDest->MemoryDependencies.push_back(BundleMember);
3896 BundleMember->Dependencies++;
3897 ScheduleData *DestBundle = DepDest->FirstInBundle;
3898 if (!DestBundle->IsScheduled) {
3899 BundleMember->incrementUnscheduledDeps(1);
3900 }
3901 if (!DestBundle->hasValidDependencies()) {
3902 WorkList.push_back(DestBundle);
3903 }
3904 }
3905 DepDest = DepDest->NextLoadStore;
3906
3907 // Example, explaining the loop break condition: Let's assume our
3908 // starting instruction is i0 and MaxMemDepDistance = 3.
3909 //
3910 // +--------v--v--v
3911 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3912 // +--------^--^--^
3913 //
3914 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3915 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3916 // Previously we already added dependencies from i3 to i6,i7,i8
3917 // (because of MaxMemDepDistance). As we added a dependency from
3918 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3919 // and we can abort this loop at i6.
3920 if (DistToSrc >= 2 * MaxMemDepDistance)
3921 break;
3922 DistToSrc++;
3923 }
3924 }
3925 }
3926 BundleMember = BundleMember->NextInBundle;
3927 }
3928 if (InsertInReadyList && SD->isReady()) {
3929 ReadyInsts.push_back(SD);
3930 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
3931 }
3932 }
3933}
3934
3935void BoUpSLP::BlockScheduling::resetSchedule() {
3936 assert(ScheduleStart &&((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3937, __PRETTY_FUNCTION__))
3937 "tried to reset schedule on block which has not been scheduled")((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3937, __PRETTY_FUNCTION__))
;
3938 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3939 doForAllOpcodes(I, [&](ScheduleData *SD) {
3940 assert(isInSchedulingRegion(SD) &&((isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3941, __PRETTY_FUNCTION__))
3941 "ScheduleData not in scheduling region")((isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3941, __PRETTY_FUNCTION__))
;
3942 SD->IsScheduled = false;
3943 SD->resetUnscheduledDeps();
3944 });
3945 }
3946 ReadyInsts.clear();
3947}
3948
3949void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3950 if (!BS->ScheduleStart)
3951 return;
3952
3953 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
3954
3955 BS->resetSchedule();
3956
3957 // For the real scheduling we use a more sophisticated ready-list: it is
3958 // sorted by the original instruction location. This lets the final schedule
3959 // be as close as possible to the original instruction order.
3960 struct ScheduleDataCompare {
3961 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3962 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3963 }
3964 };
3965 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3966
3967 // Ensure that all dependency data is updated and fill the ready-list with
3968 // initial instructions.
3969 int Idx = 0;
3970 int NumToSchedule = 0;
3971 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3972 I = I->getNextNode()) {
3973 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
3974 assert(SD->isPartOfBundle() ==((SD->isPartOfBundle() == (getTreeEntry(SD->Inst, SD->
OpValue) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst, SD->OpValue) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3976, __PRETTY_FUNCTION__))
3975 (getTreeEntry(SD->Inst, SD->OpValue) != nullptr) &&((SD->isPartOfBundle() == (getTreeEntry(SD->Inst, SD->
OpValue) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst, SD->OpValue) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3976, __PRETTY_FUNCTION__))
3976 "scheduler and vectorizer bundle mismatch")((SD->isPartOfBundle() == (getTreeEntry(SD->Inst, SD->
OpValue) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst, SD->OpValue) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3976, __PRETTY_FUNCTION__))
;
3977 SD->FirstInBundle->SchedulingPriority = Idx++;
3978 if (SD->isSchedulingEntity()) {
3979 BS->calculateDependencies(SD, false, this);
3980 NumToSchedule++;
3981 }
3982 });
3983 }
3984 BS->initialFillReadyList(ReadyInsts);
3985
3986 Instruction *LastScheduledInst = BS->ScheduleEnd;
3987
3988 // Do the "real" scheduling.
3989 while (!ReadyInsts.empty()) {
3990 ScheduleData *picked = *ReadyInsts.begin();
3991 ReadyInsts.erase(ReadyInsts.begin());
3992
3993 // Move the scheduled instruction(s) to their dedicated places, if not
3994 // there yet.
3995 ScheduleData *BundleMember = picked;
3996 while (BundleMember) {
3997 Instruction *pickedInst = BundleMember->Inst;
3998 if (pickedInst == BundleMember->OpValue) {
3999 if (LastScheduledInst->getNextNode() != pickedInst) {
4000 BS->BB->getInstList().remove(pickedInst);
4001 BS->BB->getInstList().insert(LastScheduledInst->getIterator(), pickedInst);
4002 }
4003 LastScheduledInst = pickedInst;
4004 }
4005 BundleMember = BundleMember->NextInBundle;
4006 }
4007
4008 BS->schedule(picked, ReadyInsts);
4009 NumToSchedule--;
4010 }
4011 assert(NumToSchedule == 0 && "could not schedule all instructions")((NumToSchedule == 0 && "could not schedule all instructions"
) ? static_cast<void> (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4011, __PRETTY_FUNCTION__))
;
4012
4013 // Avoid duplicate scheduling of the block.
4014 BS->ScheduleStart = nullptr;
4015}
4016
4017unsigned BoUpSLP::getVectorElementSize(Value *V) {
4018 // If V is a store, just return the width of the stored value without
4019 // traversing the expression tree. This is the common case.
4020 if (auto *Store = dyn_cast<StoreInst>(V))
4021 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4022
4023 // If V is not a store, we can traverse the expression tree to find loads
4024 // that feed it. The type of the loaded value may indicate a more suitable
4025 // width than V's type. We want to base the vector element size on the width
4026 // of memory operations where possible.
4027 SmallVector<Instruction *, 16> Worklist;
4028 SmallPtrSet<Instruction *, 16> Visited;
4029 if (auto *I = dyn_cast<Instruction>(V))
4030 Worklist.push_back(I);
4031
4032 // Traverse the expression tree in bottom-up order looking for loads. If we
4033 // encounter an instruciton we don't yet handle, we give up.
4034 auto MaxWidth = 0u;
4035 auto FoundUnknownInst = false;
4036 while (!Worklist.empty() && !FoundUnknownInst) {
4037 auto *I = Worklist.pop_back_val();
4038 Visited.insert(I);
4039
4040 // We should only be looking at scalar instructions here. If the current
4041 // instruction has a vector type, give up.
4042 auto *Ty = I->getType();
4043 if (isa<VectorType>(Ty))
4044 FoundUnknownInst = true;
4045
4046 // If the current instruction is a load, update MaxWidth to reflect the
4047 // width of the loaded value.
4048 else if (isa<LoadInst>(I))
4049 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4050
4051 // Otherwise, we need to visit the operands of the instruction. We only
4052 // handle the interesting cases from buildTree here. If an operand is an
4053 // instruction we haven't yet visited, we add it to the worklist.
4054 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4055 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4056 for (Use &U : I->operands())
4057 if (auto *J = dyn_cast<Instruction>(U.get()))
4058 if (!Visited.count(J))
4059 Worklist.push_back(J);
4060 }
4061
4062 // If we don't yet handle the instruction, give up.
4063 else
4064 FoundUnknownInst = true;
4065 }
4066
4067 // If we didn't encounter a memory access in the expression tree, or if we
4068 // gave up for some reason, just return the width of V.
4069 if (!MaxWidth || FoundUnknownInst)
4070 return DL->getTypeSizeInBits(V->getType());
4071
4072 // Otherwise, return the maximum width we found.
4073 return MaxWidth;
4074}
4075
4076// Determine if a value V in a vectorizable expression Expr can be demoted to a
4077// smaller type with a truncation. We collect the values that will be demoted
4078// in ToDemote and additional roots that require investigating in Roots.
4079static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
4080 SmallVectorImpl<Value *> &ToDemote,
4081 SmallVectorImpl<Value *> &Roots) {
4082 // We can always demote constants.
4083 if (isa<Constant>(V)) {
4084 ToDemote.push_back(V);
4085 return true;
4086 }
4087
4088 // If the value is not an instruction in the expression with only one use, it
4089 // cannot be demoted.
4090 auto *I = dyn_cast<Instruction>(V);
4091 if (!I || !I->hasOneUse() || !Expr.count(I))
4092 return false;
4093
4094 switch (I->getOpcode()) {
4095
4096 // We can always demote truncations and extensions. Since truncations can
4097 // seed additional demotion, we save the truncated value.
4098 case Instruction::Trunc:
4099 Roots.push_back(I->getOperand(0));
4100 case Instruction::ZExt:
4101 case Instruction::SExt:
4102 break;
4103
4104 // We can demote certain binary operations if we can demote both of their
4105 // operands.
4106 case Instruction::Add:
4107 case Instruction::Sub:
4108 case Instruction::Mul:
4109 case Instruction::And:
4110 case Instruction::Or:
4111 case Instruction::Xor:
4112 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4113 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4114 return false;
4115 break;
4116
4117 // We can demote selects if we can demote their true and false values.
4118 case Instruction::Select: {
4119 SelectInst *SI = cast<SelectInst>(I);
4120 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4121 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4122 return false;
4123 break;
4124 }
4125
4126 // We can demote phis if we can demote all their incoming operands. Note that
4127 // we don't need to worry about cycles since we ensure single use above.
4128 case Instruction::PHI: {
4129 PHINode *PN = cast<PHINode>(I);
4130 for (Value *IncValue : PN->incoming_values())
4131 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4132 return false;
4133 break;
4134 }
4135
4136 // Otherwise, conservatively give up.
4137 default:
4138 return false;
4139 }
4140
4141 // Record the value that we can demote.
4142 ToDemote.push_back(V);
4143 return true;
4144}
4145
4146void BoUpSLP::computeMinimumValueSizes() {
4147 // If there are no external uses, the expression tree must be rooted by a
4148 // store. We can't demote in-memory values, so there is nothing to do here.
4149 if (ExternalUses.empty())
4150 return;
4151
4152 // We only attempt to truncate integer expressions.
4153 auto &TreeRoot = VectorizableTree[0].Scalars;
4154 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4155 if (!TreeRootIT)
4156 return;
4157
4158 // If the expression is not rooted by a store, these roots should have
4159 // external uses. We will rely on InstCombine to rewrite the expression in
4160 // the narrower type. However, InstCombine only rewrites single-use values.
4161 // This means that if a tree entry other than a root is used externally, it
4162 // must have multiple uses and InstCombine will not rewrite it. The code
4163 // below ensures that only the roots are used externally.
4164 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4165 for (auto &EU : ExternalUses)
4166 if (!Expr.erase(EU.Scalar))
4167 return;
4168 if (!Expr.empty())
4169 return;
4170
4171 // Collect the scalar values of the vectorizable expression. We will use this
4172 // context to determine which values can be demoted. If we see a truncation,
4173 // we mark it as seeding another demotion.
4174 for (auto &Entry : VectorizableTree)
4175 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4176
4177 // Ensure the roots of the vectorizable tree don't form a cycle. They must
4178 // have a single external user that is not in the vectorizable tree.
4179 for (auto *Root : TreeRoot)
4180 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4181 return;
4182
4183 // Conservatively determine if we can actually truncate the roots of the
4184 // expression. Collect the values that can be demoted in ToDemote and
4185 // additional roots that require investigating in Roots.
4186 SmallVector<Value *, 32> ToDemote;
4187 SmallVector<Value *, 4> Roots;
4188 for (auto *Root : TreeRoot)
4189 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4190 return;
4191
4192 // The maximum bit width required to represent all the values that can be
4193 // demoted without loss of precision. It would be safe to truncate the roots
4194 // of the expression to this width.
4195 auto MaxBitWidth = 8u;
4196
4197 // We first check if all the bits of the roots are demanded. If they're not,
4198 // we can truncate the roots to this narrower type.
4199 for (auto *Root : TreeRoot) {
4200 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4201 MaxBitWidth = std::max<unsigned>(
4202 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4203 }
4204
4205 // True if the roots can be zero-extended back to their original type, rather
4206 // than sign-extended. We know that if the leading bits are not demanded, we
4207 // can safely zero-extend. So we initialize IsKnownPositive to True.
4208 bool IsKnownPositive = true;
4209
4210 // If all the bits of the roots are demanded, we can try a little harder to
4211 // compute a narrower type. This can happen, for example, if the roots are
4212 // getelementptr indices. InstCombine promotes these indices to the pointer
4213 // width. Thus, all their bits are technically demanded even though the
4214 // address computation might be vectorized in a smaller type.
4215 //
4216 // We start by looking at each entry that can be demoted. We compute the
4217 // maximum bit width required to store the scalar by using ValueTracking to
4218 // compute the number of high-order bits we can truncate.
4219 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
4220 MaxBitWidth = 8u;
4221
4222 // Determine if the sign bit of all the roots is known to be zero. If not,
4223 // IsKnownPositive is set to False.
4224 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4225 KnownBits Known = computeKnownBits(R, *DL);
4226 return Known.isNonNegative();
4227 });
4228
4229 // Determine the maximum number of bits required to store the scalar
4230 // values.
4231 for (auto *Scalar : ToDemote) {
4232 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4233 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4234 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4235 }
4236
4237 // If we can't prove that the sign bit is zero, we must add one to the
4238 // maximum bit width to account for the unknown sign bit. This preserves
4239 // the existing sign bit so we can safely sign-extend the root back to the
4240 // original type. Otherwise, if we know the sign bit is zero, we will
4241 // zero-extend the root instead.
4242 //
4243 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4244 // one to the maximum bit width will yield a larger-than-necessary
4245 // type. In general, we need to add an extra bit only if we can't
4246 // prove that the upper bit of the original type is equal to the
4247 // upper bit of the proposed smaller type. If these two bits are the
4248 // same (either zero or one) we know that sign-extending from the
4249 // smaller type will result in the same value. Here, since we can't
4250 // yet prove this, we are just making the proposed smaller type
4251 // larger to ensure correctness.
4252 if (!IsKnownPositive)
4253 ++MaxBitWidth;
4254 }
4255
4256 // Round MaxBitWidth up to the next power-of-two.
4257 if (!isPowerOf2_64(MaxBitWidth))
4258 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4259
4260 // If the maximum bit width we compute is less than the with of the roots'
4261 // type, we can proceed with the narrowing. Otherwise, do nothing.
4262 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4263 return;
4264
4265 // If we can truncate the root, we must collect additional values that might
4266 // be demoted as a result. That is, those seeded by truncations we will
4267 // modify.
4268 while (!Roots.empty())
4269 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4270
4271 // Finally, map the values we can demote to the maximum bit with we computed.
4272 for (auto *Scalar : ToDemote)
4273 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4274}
4275
4276namespace {
4277
4278/// The SLPVectorizer Pass.
4279struct SLPVectorizer : public FunctionPass {
4280 SLPVectorizerPass Impl;
4281
4282 /// Pass identification, replacement for typeid
4283 static char ID;
4284
4285 explicit SLPVectorizer() : FunctionPass(ID) {
4286 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4287 }
4288
4289 bool doInitialization(Module &M) override {
4290 return false;
4291 }
4292
4293 bool runOnFunction(Function &F) override {
4294 if (skipFunction(F))
4295 return false;
4296
4297 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4298 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4299 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4300 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4301 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4302 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4303 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4304 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4305 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4306 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4307
4308 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4309 }
4310
4311 void getAnalysisUsage(AnalysisUsage &AU) const override {
4312 FunctionPass::getAnalysisUsage(AU);
4313 AU.addRequired<AssumptionCacheTracker>();
4314 AU.addRequired<ScalarEvolutionWrapperPass>();
4315 AU.addRequired<AAResultsWrapperPass>();
4316 AU.addRequired<TargetTransformInfoWrapperPass>();
4317 AU.addRequired<LoopInfoWrapperPass>();
4318 AU.addRequired<DominatorTreeWrapperPass>();
4319 AU.addRequired<DemandedBitsWrapperPass>();
4320 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4321 AU.addPreserved<LoopInfoWrapperPass>();
4322 AU.addPreserved<DominatorTreeWrapperPass>();
4323 AU.addPreserved<AAResultsWrapperPass>();
4324 AU.addPreserved<GlobalsAAWrapperPass>();
4325 AU.setPreservesCFG();
4326 }
4327};
4328
4329} // end anonymous namespace
4330
4331PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4332 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4333 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4334 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4335 auto *AA = &AM.getResult<AAManager>(F);
4336 auto *LI = &AM.getResult<LoopAnalysis>(F);
4337 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4338 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4339 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4340 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4341
4342 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4343 if (!Changed)
4344 return PreservedAnalyses::all();
4345
4346 PreservedAnalyses PA;
4347 PA.preserveSet<CFGAnalyses>();
4348 PA.preserve<AAManager>();
4349 PA.preserve<GlobalsAA>();
4350 return PA;
4351}
4352
4353bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4354 TargetTransformInfo *TTI_,
4355 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4356 LoopInfo *LI_, DominatorTree *DT_,
4357 AssumptionCache *AC_, DemandedBits *DB_,
4358 OptimizationRemarkEmitter *ORE_) {
4359 SE = SE_;
4360 TTI = TTI_;
4361 TLI = TLI_;
4362 AA = AA_;
4363 LI = LI_;
4364 DT = DT_;
4365 AC = AC_;
4366 DB = DB_;
4367 DL = &F.getParent()->getDataLayout();
4368
4369 Stores.clear();
4370 GEPs.clear();
4371 bool Changed = false;
4372
4373 // If the target claims to have no vector registers don't attempt
4374 // vectorization.
4375 if (!TTI->getNumberOfRegisters(true))
4376 return false;
4377
4378 // Don't vectorize when the attribute NoImplicitFloat is used.
4379 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4380 return false;
4381
4382 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
4383
4384 // Use the bottom up slp vectorizer to construct chains that start with
4385 // store instructions.
4386 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4387
4388 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4389 // delete instructions.
4390
4391 // Scan the blocks in the function in post order.
4392 for (auto BB : post_order(&F.getEntryBlock())) {
4393 collectSeedInstructions(BB);
4394
4395 // Vectorize trees that end at stores.
4396 if (!Stores.empty()) {
4397 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
4398 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
4399 Changed |= vectorizeStoreChains(R);
4400 }
4401
4402 // Vectorize trees that end at reductions.
4403 Changed |= vectorizeChainsInBlock(BB, R);
4404
4405 // Vectorize the index computations of getelementptr instructions. This
4406 // is primarily intended to catch gather-like idioms ending at
4407 // non-consecutive loads.
4408 if (!GEPs.empty()) {
4409 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
4410 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
4411 Changed |= vectorizeGEPIndices(BB, R);
4412 }
4413 }
4414
4415 if (Changed) {
4416 R.optimizeGatherSequence();
4417 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
4418 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
4419 }
4420 return Changed;
4421}
4422
4423/// \brief Check that the Values in the slice in VL array are still existent in
4424/// the WeakTrackingVH array.
4425/// Vectorization of part of the VL array may cause later values in the VL array
4426/// to become invalid. We track when this has happened in the WeakTrackingVH
4427/// array.
4428static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4429 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4430 unsigned SliceSize) {
4431 VL = VL.slice(SliceBegin, SliceSize);
4432 VH = VH.slice(SliceBegin, SliceSize);
4433 return !std::equal(VL.begin(), VL.end(), VH.begin());
4434}
4435
4436bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4437 unsigned VecRegSize) {
4438 unsigned ChainLen = Chain.size();
4439 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
4440 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
4441 unsigned Sz = R.getVectorElementSize(Chain[0]);
4442 unsigned VF = VecRegSize / Sz;
4443
4444 if (!isPowerOf2_32(Sz) || VF < 2)
4445 return false;
4446
4447 // Keep track of values that were deleted by vectorizing in the loop below.
4448 SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4449
4450 bool Changed = false;
4451 // Look for profitable vectorizable trees at all offsets, starting at zero.
4452 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
4453 if (i + VF > e)
4454 break;
4455
4456 // Check that a previous iteration of this loop did not delete the Value.
4457 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4458 continue;
4459
4460 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
4461 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
4462 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4463
4464 R.buildTree(Operands);
4465 if (R.isTreeTinyAndNotFullyVectorizable())
4466 continue;
4467
4468 R.computeMinimumValueSizes();
4469
4470 int Cost = R.getTreeCost();
4471
4472 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
4473 if (Cost < -SLPCostThreshold) {
4474 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
4475
4476 using namespace ore;
4477
4478 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "StoresVectorized",
4479 cast<StoreInst>(Chain[i]))
4480 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4481 << " and with tree size "
4482 << NV("TreeSize", R.getTreeSize()));
4483
4484 R.vectorizeTree();
4485
4486 // Move to the next bundle.
4487 i += VF - 1;
4488 Changed = true;
4489 }
4490 }
4491
4492 return Changed;
4493}
4494
4495bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4496 BoUpSLP &R) {
4497 SetVector<StoreInst *> Heads;
4498 SmallDenseSet<StoreInst *> Tails;
4499 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4500
4501 // We may run into multiple chains that merge into a single chain. We mark the
4502 // stores that we vectorized so that we don't visit the same store twice.
4503 BoUpSLP::ValueSet VectorizedStores;
4504 bool Changed = false;
4505
4506 // Do a quadratic search on all of the given stores in reverse order and find
4507 // all of the pairs of stores that follow each other.
4508 SmallVector<unsigned, 16> IndexQueue;
4509 unsigned E = Stores.size();
4510 IndexQueue.resize(E - 1);
4511 for (unsigned I = E; I > 0; --I) {
4512 unsigned Idx = I - 1;
4513 // If a store has multiple consecutive store candidates, search Stores
4514 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4515 // This is because usually pairing with immediate succeeding or preceding
4516 // candidate create the best chance to find slp vectorization opportunity.
4517 unsigned Offset = 1;
4518 unsigned Cnt = 0;
4519 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4520 if (Idx >= Offset) {
4521 IndexQueue[Cnt] = Idx - Offset;
4522 ++Cnt;
4523 }
4524 if (Idx + Offset < E) {
4525 IndexQueue[Cnt] = Idx + Offset;
4526 ++Cnt;
4527 }
4528 }
4529
4530 for (auto K : IndexQueue) {
4531 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4532 Tails.insert(Stores[Idx]);
4533 Heads.insert(Stores[K]);
4534 ConsecutiveChain[Stores[K]] = Stores[Idx];
4535 break;
4536 }
4537 }
4538 }
4539
4540 // For stores that start but don't end a link in the chain:
4541 for (auto *SI : llvm::reverse(Heads)) {
4542 if (Tails.count(SI))
4543 continue;
4544
4545 // We found a store instr that starts a chain. Now follow the chain and try
4546 // to vectorize it.
4547 BoUpSLP::ValueList Operands;
4548 StoreInst *I = SI;
4549 // Collect the chain into a list.
4550 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4551 Operands.push_back(I);
4552 // Move to the next value in the chain.
4553 I = ConsecutiveChain[I];
4554 }
4555
4556 // FIXME: Is division-by-2 the correct step? Should we assert that the
4557 // register size is a power-of-2?
4558 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4559 Size /= 2) {
4560 if (vectorizeStoreChain(Operands, R, Size)) {
4561 // Mark the vectorized stores so that we don't vectorize them again.
4562 VectorizedStores.insert(Operands.begin(), Operands.end());
4563 Changed = true;
4564 break;
4565 }
4566 }
4567 }
4568
4569 return Changed;
4570}
4571
4572void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4573 // Initialize the collections. We will make a single pass over the block.
4574 Stores.clear();
4575 GEPs.clear();
4576
4577 // Visit the store and getelementptr instructions in BB and organize them in
4578 // Stores and GEPs according to the underlying objects of their pointer
4579 // operands.
4580 for (Instruction &I : *BB) {
4581 // Ignore store instructions that are volatile or have a pointer operand
4582 // that doesn't point to a scalar type.
4583 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4584 if (!SI->isSimple())
4585 continue;
4586 if (!isValidElementType(SI->getValueOperand()->getType()))
4587 continue;
4588 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4589 }
4590
4591 // Ignore getelementptr instructions that have more than one index, a
4592 // constant index, or a pointer operand that doesn't point to a scalar
4593 // type.
4594 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4595 auto Idx = GEP->idx_begin()->get();
4596 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4597 continue;
4598 if (!isValidElementType(Idx->getType()))
4599 continue;
4600 if (GEP->getType()->isVectorTy())
4601 continue;
4602 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4603 }
4604 }
4605}
4606
4607bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4608 if (!A || !B)
4609 return false;
4610 Value *VL[] = { A, B };
4611 return tryToVectorizeList(VL, R, None, true);
4612}
4613
4614bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4615 ArrayRef<Value *> BuildVector,
4616 bool AllowReorder) {
4617 if (VL.size() < 2)
4618 return false;
4619
4620 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4621 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
4622
4623 // Check that all of the parts are scalar instructions of the same type.
4624 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4625 if (!I0)
4626 return false;
4627
4628 unsigned Opcode0 = I0->getOpcode();
4629
4630 unsigned Sz = R.getVectorElementSize(I0);
4631 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4632 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4633 if (MaxVF < 2)
4634 return false;
4635
4636 for (Value *V : VL) {
4637 Type *Ty = V->getType();
4638 if (!isValidElementType(Ty))
4639 return false;
4640 Instruction *Inst = dyn_cast<Instruction>(V);
4641 if (!Inst || Inst->getOpcode() != Opcode0)
4642 return false;
4643 }
4644
4645 bool Changed = false;
4646
4647 // Keep track of values that were deleted by vectorizing in the loop below.
4648 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4649
4650 unsigned NextInst = 0, MaxInst = VL.size();
4651 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4652 VF /= 2) {
4653 // No actual vectorization should happen, if number of parts is the same as
4654 // provided vectorization factor (i.e. the scalar type is used for vector
4655 // code during codegen).
4656 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4657 if (TTI->getNumberOfParts(VecTy) == VF)
4658 continue;
4659 for (unsigned I = NextInst; I < MaxInst; ++I) {
4660 unsigned OpsWidth = 0;
4661
4662 if (I + VF > MaxInst)
4663 OpsWidth = MaxInst - I;
4664 else
4665 OpsWidth = VF;
4666
4667 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4668 break;
4669
4670 // Check that a previous iteration of this loop did not delete the Value.
4671 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4672 continue;
4673
4674 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4675 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
4676 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4677
4678 ArrayRef<Value *> BuildVectorSlice;
4679 if (!BuildVector.empty())
4680 BuildVectorSlice = BuildVector.slice(I, OpsWidth);
4681
4682 R.buildTree(Ops, BuildVectorSlice);
4683 // TODO: check if we can allow reordering for more cases.
4684 if (AllowReorder && R.shouldReorder()) {
4685 // Conceptually, there is nothing actually preventing us from trying to
4686 // reorder a larger list. In fact, we do exactly this when vectorizing
4687 // reductions. However, at this point, we only expect to get here when
4688 // there are exactly two operations.
4689 assert(Ops.size() == 2)((Ops.size() == 2) ? static_cast<void> (0) : __assert_fail
("Ops.size() == 2", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4689, __PRETTY_FUNCTION__))
;
4690 assert(BuildVectorSlice.empty())((BuildVectorSlice.empty()) ? static_cast<void> (0) : __assert_fail
("BuildVectorSlice.empty()", "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4690, __PRETTY_FUNCTION__))
;
4691 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4692 R.buildTree(ReorderedOps, None);
4693 }
4694 if (R.isTreeTinyAndNotFullyVectorizable())
4695 continue;
4696
4697 R.computeMinimumValueSizes();
4698 int Cost = R.getTreeCost();
4699
4700 if (Cost < -SLPCostThreshold) {
4701 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
4702 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedList",
4703 cast<Instruction>(Ops[0]))
4704 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4705 << " and with tree size "
4706 << ore::NV("TreeSize", R.getTreeSize()));
4707
4708 Value *VectorizedRoot = R.vectorizeTree();
4709
4710 // Reconstruct the build vector by extracting the vectorized root. This
4711 // way we handle the case where some elements of the vector are
4712 // undefined.
4713 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
4714 if (!BuildVectorSlice.empty()) {
4715 // The insert point is the last build vector instruction. The
4716 // vectorized root will precede it. This guarantees that we get an
4717 // instruction. The vectorized tree could have been constant folded.
4718 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4719 unsigned VecIdx = 0;
4720 for (auto &V : BuildVectorSlice) {
4721 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4722 ++BasicBlock::iterator(InsertAfter));
4723 Instruction *I = cast<Instruction>(V);
4724 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I))((isa<InsertElementInst>(I) || isa<InsertValueInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<InsertElementInst>(I) || isa<InsertValueInst>(I)"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4724, __PRETTY_FUNCTION__))
;
4725 Instruction *Extract =
4726 cast<Instruction>(Builder.CreateExtractElement(
4727 VectorizedRoot, Builder.getInt32(VecIdx++)));
4728 I->setOperand(1, Extract);
4729 I->moveAfter(Extract);
4730 InsertAfter = I;
4731 }
4732 }
4733 // Move to the next bundle.
4734 I += VF - 1;
4735 NextInst = I + 1;
4736 Changed = true;
4737 }
4738 }
4739 }
4740
4741 return Changed;
4742}
4743
4744bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4745 if (!I)
4746 return false;
4747
4748 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4749 return false;
4750
4751 Value *P = I->getParent();
4752
4753 // Vectorize in current basic block only.
4754 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4755 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4756 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4757 return false;
4758
4759 // Try to vectorize V.
4760 if (tryToVectorizePair(Op0, Op1, R))
4761 return true;
4762
4763 auto *A = dyn_cast<BinaryOperator>(Op0);
4764 auto *B = dyn_cast<BinaryOperator>(Op1);
4765 // Try to skip B.
4766 if (B && B->hasOneUse()) {
4767 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4768 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4769 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4770 return true;
4771 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4772 return true;
4773 }
4774
4775 // Try to skip A.
4776 if (A && A->hasOneUse()) {
4777 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4778 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4779 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4780 return true;
4781 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4782 return true;
4783 }
4784 return false;
4785}
4786
4787/// \brief Generate a shuffle mask to be used in a reduction tree.
4788///
4789/// \param VecLen The length of the vector to be reduced.
4790/// \param NumEltsToRdx The number of elements that should be reduced in the
4791/// vector.
4792/// \param IsPairwise Whether the reduction is a pairwise or splitting
4793/// reduction. A pairwise reduction will generate a mask of
4794/// <0,2,...> or <1,3,..> while a splitting reduction will generate
4795/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4796/// \param IsLeft True will generate a mask of even elements, odd otherwise.
4797static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4798 bool IsPairwise, bool IsLeft,
4799 IRBuilder<> &Builder) {
4800 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"
) ? static_cast<void> (0) : __assert_fail ("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4800, __PRETTY_FUNCTION__))
;
4801
4802 SmallVector<Constant *, 32> ShuffleMask(
4803 VecLen, UndefValue::get(Builder.getInt32Ty()));
4804
4805 if (IsPairwise)
4806 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4807 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4808 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4809 else
4810 // Move the upper half of the vector to the lower half.
4811 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4812 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4813
4814 return ConstantVector::get(ShuffleMask);
4815}
4816
4817namespace {
4818
4819/// Model horizontal reductions.
4820///
4821/// A horizontal reduction is a tree of reduction operations (currently add and
4822/// fadd) that has operations that can be put into a vector as its leaf.
4823/// For example, this tree:
4824///
4825/// mul mul mul mul
4826/// \ / \ /
4827/// + +
4828/// \ /
4829/// +
4830/// This tree has "mul" as its reduced values and "+" as its reduction
4831/// operations. A reduction might be feeding into a store or a binary operation
4832/// feeding a phi.
4833/// ...
4834/// \ /
4835/// +
4836/// |
4837/// phi +=
4838///
4839/// Or:
4840/// ...
4841/// \ /
4842/// +
4843/// |
4844/// *p =
4845///
4846class HorizontalReduction {
4847 using ReductionOpsType = SmallVector<Value *, 16>;
4848 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
4849 ReductionOpsListType ReductionOps;
4850 SmallVector<Value *, 32> ReducedVals;
4851 // Use map vector to make stable output.
4852 MapVector<Instruction *, Value *> ExtraArgs;
4853
4854 /// Kind of the reduction data.
4855 enum ReductionKind {
4856 RK_None, /// Not a reduction.
4857 RK_Arithmetic, /// Binary reduction data.
4858 RK_Min, /// Minimum reduction data.
4859 RK_UMin, /// Unsigned minimum reduction data.
4860 RK_Max, /// Maximum reduction data.
4861 RK_UMax, /// Unsigned maximum reduction data.
4862 };
4863
4864 /// Contains info about operation, like its opcode, left and right operands.
4865 class OperationData {
4866 /// Opcode of the instruction.
4867 unsigned Opcode = 0;
4868
4869 /// Left operand of the reduction operation.
4870 Value *LHS = nullptr;
4871
4872 /// Right operand of the reduction operation.
4873 Value *RHS = nullptr;
4874
4875 /// Kind of the reduction operation.
4876 ReductionKind Kind = RK_None;
4877
4878 /// True if float point min/max reduction has no NaNs.
4879 bool NoNaN = false;
4880
4881 /// Checks if the reduction operation can be vectorized.
4882 bool isVectorizable() const {
4883 return LHS && RHS &&
4884 // We currently only support adds && min/max reductions.
4885 ((Kind == RK_Arithmetic &&
4886 (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
4887 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
4888 (Kind == RK_Min || Kind == RK_Max)) ||
4889 (Opcode == Instruction::ICmp &&
4890 (Kind == RK_UMin || Kind == RK_UMax)));
4891 }
4892
4893 /// Creates reduction operation with the current opcode.
4894 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
4895 assert(isVectorizable() &&((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4896, __PRETTY_FUNCTION__))
4896 "Expected add|fadd or min/max reduction operation.")((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4896, __PRETTY_FUNCTION__))
;
4897 Value *Cmp;
4898 switch (Kind) {
4899 case RK_Arithmetic:
4900 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
4901 Name);
4902 case RK_Min:
4903 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
4904 : Builder.CreateFCmpOLT(LHS, RHS);
4905 break;
4906 case RK_Max:
4907 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
4908 : Builder.CreateFCmpOGT(LHS, RHS);
4909 break;
4910 case RK_UMin:
4911 assert(Opcode == Instruction::ICmp && "Expected integer types.")((Opcode == Instruction::ICmp && "Expected integer types."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4911, __PRETTY_FUNCTION__))
;
4912 Cmp = Builder.CreateICmpULT(LHS, RHS);
4913 break;
4914 case RK_UMax:
4915 assert(Opcode == Instruction::ICmp && "Expected integer types.")((Opcode == Instruction::ICmp && "Expected integer types."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4915, __PRETTY_FUNCTION__))
;
4916 Cmp = Builder.CreateICmpUGT(LHS, RHS);
4917 break;
4918 case RK_None:
4919 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4919)
;
4920 }
4921 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
4922 }
4923
4924 public:
4925 explicit OperationData() = default;
4926
4927 /// Construction for reduced values. They are identified by opcode only and
4928 /// don't have associated LHS/RHS values.
4929 explicit OperationData(Value *V) {
4930 if (auto *I = dyn_cast<Instruction>(V))
4931 Opcode = I->getOpcode();
4932 }
4933
4934 /// Constructor for reduction operations with opcode and its left and
4935 /// right operands.
4936 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
4937 bool NoNaN = false)
4938 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
4939 assert(Kind != RK_None && "One of the reduction operations is expected.")((Kind != RK_None && "One of the reduction operations is expected."
) ? static_cast<void> (0) : __assert_fail ("Kind != RK_None && \"One of the reduction operations is expected.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4939, __PRETTY_FUNCTION__))
;
4940 }
4941
4942 explicit operator bool() const { return Opcode; }
4943
4944 /// Get the index of the first operand.
4945 unsigned getFirstOperandIndex() const {
4946 assert(!!*this && "The opcode is not set.")((!!*this && "The opcode is not set.") ? static_cast<
void> (0) : __assert_fail ("!!*this && \"The opcode is not set.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4946, __PRETTY_FUNCTION__))
;
4947 switch (Kind) {
4948 case RK_Min:
4949 case RK_UMin:
4950 case RK_Max:
4951 case RK_UMax:
4952 return 1;
4953 case RK_Arithmetic:
4954 case RK_None:
4955 break;
4956 }
4957 return 0;
4958 }
4959
4960 /// Total number of operands in the reduction operation.
4961 unsigned getNumberOfOperands() const {
4962 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4963, __PRETTY_FUNCTION__))
4963 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4963, __PRETTY_FUNCTION__))
;
4964 switch (Kind) {
4965 case RK_Arithmetic:
4966 return 2;
4967 case RK_Min:
4968 case RK_UMin:
4969 case RK_Max:
4970 case RK_UMax:
4971 return 3;
4972 case RK_None:
4973 break;
4974 }
4975 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4975)
;
4976 }
4977
4978 /// Checks if the operation has the same parent as \p P.
4979 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
4980 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4981, __PRETTY_FUNCTION__))
4981 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4981, __PRETTY_FUNCTION__))
;
4982 if (!IsRedOp)
4983 return I->getParent() == P;
4984 switch (Kind) {
4985 case RK_Arithmetic:
4986 // Arithmetic reduction operation must be used once only.
4987 return I->getParent() == P;
4988 case RK_Min:
4989 case RK_UMin:
4990 case RK_Max:
4991 case RK_UMax: {
4992 // SelectInst must be used twice while the condition op must have single
4993 // use only.
4994 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
4995 return I->getParent() == P && Cmp && Cmp->getParent() == P;
4996 }
4997 case RK_None:
4998 break;
4999 }
5000 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5000)
;
5001 }
5002 /// Expected number of uses for reduction operations/reduced values.
5003 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5004 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5005, __PRETTY_FUNCTION__))
5005 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5005, __PRETTY_FUNCTION__))
;
5006 switch (Kind) {
5007 case RK_Arithmetic:
5008 return I->hasOneUse();
5009 case RK_Min:
5010 case RK_UMin:
5011 case RK_Max:
5012 case RK_UMax:
5013 return I->hasNUses(2) &&
5014 (!IsReductionOp ||
5015 cast<SelectInst>(I)->getCondition()->hasOneUse());
5016 case RK_None:
5017 break;
5018 }
5019 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5019)
;
5020 }
5021
5022 /// Initializes the list of reduction operations.
5023 void initReductionOps(ReductionOpsListType &ReductionOps) {
5024 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5025, __PRETTY_FUNCTION__))
5025 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5025, __PRETTY_FUNCTION__))
;
5026 switch (Kind) {
5027 case RK_Arithmetic:
5028 ReductionOps.assign(1, ReductionOpsType());
5029 break;
5030 case RK_Min:
5031 case RK_UMin:
5032 case RK_Max:
5033 case RK_UMax:
5034 ReductionOps.assign(2, ReductionOpsType());
5035 break;
5036 case RK_None:
5037 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5037)
;
5038 }
5039 }
5040 /// Add all reduction operations for the reduction instruction \p I.
5041 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5042 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5043, __PRETTY_FUNCTION__))
5043 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5043, __PRETTY_FUNCTION__))
;
5044 switch (Kind) {
5045 case RK_Arithmetic:
5046 ReductionOps[0].emplace_back(I);
5047 break;
5048 case RK_Min:
5049 case RK_UMin:
5050 case RK_Max:
5051 case RK_UMax:
5052 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5053 ReductionOps[1].emplace_back(I);
5054 break;
5055 case RK_None:
5056 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5056)
;
5057 }
5058 }
5059
5060 /// Checks if instruction is associative and can be vectorized.
5061 bool isAssociative(Instruction *I) const {
5062 assert(Kind != RK_None && *this && LHS && RHS &&((Kind != RK_None && *this && LHS && RHS
&& "Expected reduction operation.") ? static_cast<
void> (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5063, __PRETTY_FUNCTION__))
5063 "Expected reduction operation.")((Kind != RK_None && *this && LHS && RHS
&& "Expected reduction operation.") ? static_cast<
void> (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5063, __PRETTY_FUNCTION__))
;
5064 switch (Kind) {
5065 case RK_Arithmetic:
5066 return I->isAssociative();
5067 case RK_Min:
5068 case RK_Max:
5069 return Opcode == Instruction::ICmp ||
5070 cast<Instruction>(I->getOperand(0))->isFast();
5071 case RK_UMin:
5072 case RK_UMax:
5073 assert(Opcode == Instruction::ICmp &&((Opcode == Instruction::ICmp && "Only integer compare operation is expected."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5074, __PRETTY_FUNCTION__))
5074 "Only integer compare operation is expected.")((Opcode == Instruction::ICmp && "Only integer compare operation is expected."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5074, __PRETTY_FUNCTION__))
;
5075 return true;
5076 case RK_None:
5077 break;
5078 }
5079 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5079)
;
5080 }
5081
5082 /// Checks if the reduction operation can be vectorized.
5083 bool isVectorizable(Instruction *I) const {
5084 return isVectorizable() && isAssociative(I);
5085 }
5086
5087 /// Checks if two operation data are both a reduction op or both a reduced
5088 /// value.
5089 bool operator==(const OperationData &OD) {
5090 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&((((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS ==
!OD.RHS))) && "One of the comparing operations is incorrect."
) ? static_cast<void> (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5091, __PRETTY_FUNCTION__))
5091 "One of the comparing operations is incorrect.")((((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS ==
!OD.RHS))) && "One of the comparing operations is incorrect."
) ? static_cast<void> (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5091, __PRETTY_FUNCTION__))
;
5092 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5093 }
5094 bool operator!=(const OperationData &OD) { return !(*this == OD); }
5095 void clear() {
5096 Opcode = 0;
5097 LHS = nullptr;
5098 RHS = nullptr;
5099 Kind = RK_None;
5100 NoNaN = false;
5101 }
5102
5103 /// Get the opcode of the reduction operation.
5104 unsigned getOpcode() const {
5105 assert(isVectorizable() && "Expected vectorizable operation.")((isVectorizable() && "Expected vectorizable operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected vectorizable operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5105, __PRETTY_FUNCTION__))
;
5106 return Opcode;
5107 }
5108
5109 /// Get kind of reduction data.
5110 ReductionKind getKind() const { return Kind; }
5111 Value *getLHS() const { return LHS; }
5112 Value *getRHS() const { return RHS; }
5113 Type *getConditionType() const {
5114 switch (Kind) {
5115 case RK_Arithmetic:
5116 return nullptr;
5117 case RK_Min:
5118 case RK_Max:
5119 case RK_UMin:
5120 case RK_UMax:
5121 return CmpInst::makeCmpResultType(LHS->getType());
5122 case RK_None:
5123 break;
5124 }
5125 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5125)
;
5126 }
5127
5128 /// Creates reduction operation with the current opcode with the IR flags
5129 /// from \p ReductionOps.
5130 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5131 const ReductionOpsListType &ReductionOps) const {
5132 assert(isVectorizable() &&((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5133, __PRETTY_FUNCTION__))
5133 "Expected add|fadd or min/max reduction operation.")((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5133, __PRETTY_FUNCTION__))
;
5134 auto *Op = createOp(Builder, Name);
5135 switch (Kind) {
5136 case RK_Arithmetic:
5137 propagateIRFlags(Op, ReductionOps[0]);
5138 return Op;
5139 case RK_Min:
5140 case RK_Max:
5141 case RK_UMin:
5142 case RK_UMax:
5143 if (auto *SI = dyn_cast<SelectInst>(Op))
5144 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5145 propagateIRFlags(Op, ReductionOps[1]);
5146 return Op;
5147 case RK_None:
5148 break;
5149 }
5150 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5150)
;
5151 }
5152 /// Creates reduction operation with the current opcode with the IR flags
5153 /// from \p I.
5154 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5155 Instruction *I) const {
5156 assert(isVectorizable() &&((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5157, __PRETTY_FUNCTION__))
5157 "Expected add|fadd or min/max reduction operation.")((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5157, __PRETTY_FUNCTION__))
;
5158 auto *Op = createOp(Builder, Name);
5159 switch (Kind) {
5160 case RK_Arithmetic:
5161 propagateIRFlags(Op, I);
5162 return Op;
5163 case RK_Min:
5164 case RK_Max:
5165 case RK_UMin:
5166 case RK_UMax:
5167 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5168 propagateIRFlags(SI->getCondition(),
5169 cast<SelectInst>(I)->getCondition());
5170 }
5171 propagateIRFlags(Op, I);
5172 return Op;
5173 case RK_None:
5174 break;
5175 }
5176 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5176)
;
5177 }
5178
5179 TargetTransformInfo::ReductionFlags getFlags() const {
5180 TargetTransformInfo::ReductionFlags Flags;
5181 Flags.NoNaN = NoNaN;
5182 switch (Kind) {
5183 case RK_Arithmetic:
5184 break;
5185 case RK_Min:
5186 Flags.IsSigned = Opcode == Instruction::ICmp;
5187 Flags.IsMaxOp = false;
5188 break;
5189 case RK_Max:
5190 Flags.IsSigned = Opcode == Instruction::ICmp;
5191 Flags.IsMaxOp = true;
5192 break;
5193 case RK_UMin:
5194 Flags.IsSigned = false;
5195 Flags.IsMaxOp = false;
5196 break;
5197 case RK_UMax:
5198 Flags.IsSigned = false;
5199 Flags.IsMaxOp = true;
5200 break;
5201 case RK_None:
5202 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5202)
;
5203 }
5204 return Flags;
5205 }
5206 };
5207
5208 Instruction *ReductionRoot = nullptr;
5209
5210 /// The operation data of the reduction operation.
5211 OperationData ReductionData;
5212
5213 /// The operation data of the values we perform a reduction on.
5214 OperationData ReducedValueData;
5215
5216 /// Should we model this reduction as a pairwise reduction tree or a tree that
5217 /// splits the vector in halves and adds those halves.
5218 bool IsPairwiseReduction = false;
5219
5220 /// Checks if the ParentStackElem.first should be marked as a reduction
5221 /// operation with an extra argument or as extra argument itself.
5222 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5223 Value *ExtraArg) {
5224 if (ExtraArgs.count(ParentStackElem.first)) {
5225 ExtraArgs[ParentStackElem.first] = nullptr;
5226 // We ran into something like:
5227 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5228 // The whole ParentStackElem.first should be considered as an extra value
5229 // in this case.
5230 // Do not perform analysis of remaining operands of ParentStackElem.first
5231 // instruction, this whole instruction is an extra argument.
5232 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5233 } else {
5234 // We ran into something like:
5235 // ParentStackElem.first += ... + ExtraArg + ...
5236 ExtraArgs[ParentStackElem.first] = ExtraArg;
5237 }
5238 }
5239
5240 static OperationData getOperationData(Value *V) {
5241 if (!V)
5242 return OperationData();
5243
5244 Value *LHS;
5245 Value *RHS;
5246 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5247 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5248 RK_Arithmetic);
5249 }
5250 if (auto *Select = dyn_cast<SelectInst>(V)) {
5251 // Look for a min/max pattern.
5252 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5253 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5254 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5255 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5256 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5257 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5258 return OperationData(
5259 Instruction::FCmp, LHS, RHS, RK_Min,
5260 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5261 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5262 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5263 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5264 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5265 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5266 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5267 return OperationData(
5268 Instruction::FCmp, LHS, RHS, RK_Max,
5269 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5270 }
5271 }
5272 return OperationData(V);
5273 }
5274
5275public:
5276 HorizontalReduction() = default;
5277
5278 /// \brief Try to find a reduction tree.
5279 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5280 assert((!Phi || is_contained(Phi->operands(), B)) &&(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5281, __PRETTY_FUNCTION__))
5281 "Thi phi needs to use the binary operator")(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5281, __PRETTY_FUNCTION__))
;
5282
5283 ReductionData = getOperationData(B);
5284
5285 // We could have a initial reductions that is not an add.
5286 // r *= v1 + v2 + v3 + v4
5287 // In such a case start looking for a tree rooted in the first '+'.
5288 if (Phi) {
5289 if (ReductionData.getLHS() == Phi) {
5290 Phi = nullptr;
5291 B = dyn_cast<Instruction>(ReductionData.getRHS());
5292 ReductionData = getOperationData(B);
5293 } else if (ReductionData.getRHS() == Phi) {
5294 Phi = nullptr;
5295 B = dyn_cast<Instruction>(ReductionData.getLHS());
5296 ReductionData = getOperationData(B);
5297 }
5298 }
5299
5300 if (!ReductionData.isVectorizable(B))
5301 return false;
5302
5303 Type *Ty = B->getType();
5304 if (!isValidElementType(Ty))
5305 return false;
5306
5307 ReducedValueData.clear();
5308 ReductionRoot = B;
5309
5310 // Post order traverse the reduction tree starting at B. We only handle true
5311 // trees containing only binary operators.
5312 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5313 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5314 ReductionData.initReductionOps(ReductionOps);
5315 while (!Stack.empty()) {
5316 Instruction *TreeN = Stack.back().first;
5317 unsigned EdgeToVist = Stack.back().second++;
5318 OperationData OpData = getOperationData(TreeN);
5319 bool IsReducedValue = OpData != ReductionData;
5320
5321 // Postorder vist.
5322 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5323 if (IsReducedValue)
5324 ReducedVals.push_back(TreeN);
5325 else {
5326 auto I = ExtraArgs.find(TreeN);
5327 if (I != ExtraArgs.end() && !I->second) {
5328 // Check if TreeN is an extra argument of its parent operation.
5329 if (Stack.size() <= 1) {
5330 // TreeN can't be an extra argument as it is a root reduction
5331 // operation.
5332 return false;
5333 }
5334 // Yes, TreeN is an extra argument, do not add it to a list of
5335 // reduction operations.
5336 // Stack[Stack.size() - 2] always points to the parent operation.
5337 markExtraArg(Stack[Stack.size() - 2], TreeN);
5338 ExtraArgs.erase(TreeN);
5339 } else
5340 ReductionData.addReductionOps(TreeN, ReductionOps);
5341 }
5342 // Retract.
5343 Stack.pop_back();
5344 continue;
5345 }
5346
5347 // Visit left or right.
5348 Value *NextV = TreeN->getOperand(EdgeToVist);
5349 if (NextV != Phi) {
5350 auto *I = dyn_cast<Instruction>(NextV);
5351 OpData = getOperationData(I);
5352 // Continue analysis if the next operand is a reduction operation or
5353 // (possibly) a reduced value. If the reduced value opcode is not set,
5354 // the first met operation != reduction operation is considered as the
5355 // reduced value class.
5356 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5357 OpData == ReductionData)) {
5358 const bool IsReductionOperation = OpData == ReductionData;
5359 // Only handle trees in the current basic block.
5360 if (!ReductionData.hasSameParent(I, B->getParent(),
5361 IsReductionOperation)) {
5362 // I is an extra argument for TreeN (its parent operation).
5363 markExtraArg(Stack.back(), I);
5364 continue;
5365 }
5366
5367 // Each tree node needs to have minimal number of users except for the
5368 // ultimate reduction.
5369 if (!ReductionData.hasRequiredNumberOfUses(I,
5370 OpData == ReductionData) &&
5371 I != B) {
5372 // I is an extra argument for TreeN (its parent operation).
5373 markExtraArg(Stack.back(), I);
5374 continue;
5375 }
5376
5377 if (IsReductionOperation) {
5378 // We need to be able to reassociate the reduction operations.
5379 if (!OpData.isAssociative(I)) {
5380 // I is an extra argument for TreeN (its parent operation).
5381 markExtraArg(Stack.back(), I);
5382 continue;
5383 }
5384 } else if (ReducedValueData &&
5385 ReducedValueData != OpData) {
5386 // Make sure that the opcodes of the operations that we are going to
5387 // reduce match.
5388 // I is an extra argument for TreeN (its parent operation).
5389 markExtraArg(Stack.back(), I);
5390 continue;
5391 } else if (!ReducedValueData)
5392 ReducedValueData = OpData;
5393
5394 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5395 continue;
5396 }
5397 }
5398 // NextV is an extra argument for TreeN (its parent operation).
5399 markExtraArg(Stack.back(), NextV);
5400 }
5401 return true;
5402 }
5403
5404 /// \brief Attempt to vectorize the tree found by
5405 /// matchAssociativeReduction.
5406 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5407 if (ReducedVals.empty())
5408 return false;
5409
5410 // If there is a sufficient number of reduction values, reduce
5411 // to a nearby power-of-2. Can safely generate oversized
5412 // vectors and rely on the backend to split them to legal sizes.
5413 unsigned NumReducedVals = ReducedVals.size();
5414 if (NumReducedVals < 4)
5415 return false;
5416
5417 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5418
5419 Value *VectorizedTree = nullptr;
5420 IRBuilder<> Builder(ReductionRoot);
5421 FastMathFlags Unsafe;
5422 Unsafe.setFast();
5423 Builder.setFastMathFlags(Unsafe);
5424 unsigned i = 0;
5425
5426 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5427 // The same extra argument may be used several time, so log each attempt
5428 // to use it.
5429 for (auto &Pair : ExtraArgs)
5430 ExternallyUsedValues[Pair.second].push_back(Pair.first);
5431 SmallVector<Value *, 16> IgnoreList;
5432 for (auto &V : ReductionOps)
5433 IgnoreList.append(V.begin(), V.end());
5434 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5435 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5436 V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5437 if (V.shouldReorder()) {
5438 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
5439 V.buildTree(Reversed, ExternallyUsedValues, IgnoreList);
5440 }
5441 if (V.isTreeTinyAndNotFullyVectorizable())
5442 break;
5443
5444 V.computeMinimumValueSizes();
5445
5446 // Estimate cost.
5447 int Cost =
5448 V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5449 if (Cost >= -SLPCostThreshold)
5450 break;
5451
5452 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
5453 << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
;
5454 auto *I0 = cast<Instruction>(VL[0]);
5455 V.getORE()->emit(
5456 OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedHorizontalReduction", I0)
5457 << "Vectorized horizontal reduction with cost "
5458 << ore::NV("Cost", Cost) << " and with tree size "
5459 << ore::NV("TreeSize", V.getTreeSize()));
5460
5461 // Vectorize a tree.
5462 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5463 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5464
5465 // Emit a reduction.
5466 Value *ReducedSubTree =
5467 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5468 if (VectorizedTree) {
5469 Builder.SetCurrentDebugLocation(Loc);
5470 OperationData VectReductionData(ReductionData.getOpcode(),
5471 VectorizedTree, ReducedSubTree,
5472 ReductionData.getKind());
5473 VectorizedTree =
5474 VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5475 } else
5476 VectorizedTree = ReducedSubTree;
5477 i += ReduxWidth;
5478 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5479 }
5480
5481 if (VectorizedTree) {
5482 // Finish the reduction.
5483 for (; i < NumReducedVals; ++i) {
5484 auto *I = cast<Instruction>(ReducedVals[i]);
5485 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5486 OperationData VectReductionData(ReductionData.getOpcode(),
5487 VectorizedTree, I,
5488 ReductionData.getKind());
5489 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5490 }
5491 for (auto &Pair : ExternallyUsedValues) {
5492 assert(!Pair.second.empty() &&((!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? static_cast<void> (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5493, __PRETTY_FUNCTION__))
5493 "At least one DebugLoc must be inserted")((!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? static_cast<void> (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5493, __PRETTY_FUNCTION__))
;
5494 // Add each externally used value to the final reduction.
5495 for (auto *I : Pair.second) {
5496 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5497 OperationData VectReductionData(ReductionData.getOpcode(),
5498 VectorizedTree, Pair.first,
5499 ReductionData.getKind());
5500 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5501 }
5502 }
5503 // Update users.
5504 ReductionRoot->replaceAllUsesWith(VectorizedTree);
5505 }
5506 return VectorizedTree != nullptr;
5507 }
5508
5509 unsigned numReductionValues() const {
5510 return ReducedVals.size();
5511 }
5512
5513private:
5514 /// \brief Calculate the cost of a reduction.
5515 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5516 unsigned ReduxWidth) {
5517 Type *ScalarTy = FirstReducedVal->getType();
5518 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5519
5520 int PairwiseRdxCost;
5521 int SplittingRdxCost;
5522 switch (ReductionData.getKind()) {
5523 case RK_Arithmetic:
5524 PairwiseRdxCost =
5525 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5526 /*IsPairwiseForm=*/true);
5527 SplittingRdxCost =
5528 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5529 /*IsPairwiseForm=*/false);
5530 break;
5531 case RK_Min:
5532 case RK_Max:
5533 case RK_UMin:
5534 case RK_UMax: {
5535 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5536 bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5537 ReductionData.getKind() == RK_UMax;
5538 PairwiseRdxCost =
5539 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5540 /*IsPairwiseForm=*/true, IsUnsigned);
5541 SplittingRdxCost =
5542 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5543 /*IsPairwiseForm=*/false, IsUnsigned);
5544 break;
5545 }
5546 case RK_None:
5547 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5547)
;
5548 }
5549
5550 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5551 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5552
5553 int ScalarReduxCost;
5554 switch (ReductionData.getKind()) {
5555 case RK_Arithmetic:
5556 ScalarReduxCost =
5557 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5558 break;
5559 case RK_Min:
5560 case RK_Max:
5561 case RK_UMin:
5562 case RK_UMax:
5563 ScalarReduxCost =
5564 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5565 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5566 CmpInst::makeCmpResultType(ScalarTy));
5567 break;
5568 case RK_None:
5569 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5569)
;
5570 }
5571 ScalarReduxCost *= (ReduxWidth - 1);
5572
5573 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5574 << " for reduction that starts with " << *FirstReducedValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5575 << " (It is a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5576 << (IsPairwiseReduction ? "pairwise" : "splitting")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5577 << " reduction)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
;
5578
5579 return VecReduxCost - ScalarReduxCost;
5580 }
5581
5582 /// \brief Emit a horizontal reduction of the vectorized value.
5583 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5584 unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5585 assert(VectorizedValue && "Need to have a vectorized tree node")((VectorizedValue && "Need to have a vectorized tree node"
) ? static_cast<void> (0) : __assert_fail ("VectorizedValue && \"Need to have a vectorized tree node\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5585, __PRETTY_FUNCTION__))
;
5586 assert(isPowerOf2_32(ReduxWidth) &&((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5587, __PRETTY_FUNCTION__))
5587 "We only handle power-of-two reductions for now")((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5587, __PRETTY_FUNCTION__))
;
5588
5589 if (!IsPairwiseReduction)
5590 return createSimpleTargetReduction(
5591 Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5592 ReductionData.getFlags(), ReductionOps.back());
5593
5594 Value *TmpVec = VectorizedValue;
5595 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5596 Value *LeftMask =
5597 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5598 Value *RightMask =
5599 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5600
5601 Value *LeftShuf = Builder.CreateShuffleVector(
5602 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5603 Value *RightShuf = Builder.CreateShuffleVector(
5604 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5605 "rdx.shuf.r");
5606 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5607 RightShuf, ReductionData.getKind());
5608 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5609 }
5610
5611 // The result is in the first element of the vector.
5612 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5613 }
5614};
5615
5616} // end anonymous namespace
5617
5618/// \brief Recognize construction of vectors like
5619/// %ra = insertelement <4 x float> undef, float %s0, i32 0
5620/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5621/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5622/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5623/// starting from the last insertelement instruction.
5624///
5625/// Returns true if it matches
5626static bool findBuildVector(InsertElementInst *LastInsertElem,
5627 SmallVectorImpl<Value *> &BuildVector,
5628 SmallVectorImpl<Value *> &BuildVectorOpds) {
5629 Value *V = nullptr;
5630 do {
5631 BuildVector.push_back(LastInsertElem);
5632 BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5633 V = LastInsertElem->getOperand(0);
5634 if (isa<UndefValue>(V))
5635 break;
5636 LastInsertElem = dyn_cast<InsertElementInst>(V);
5637 if (!LastInsertElem || !LastInsertElem->hasOneUse())
5638 return false;
5639 } while (true);
5640 std::reverse(BuildVector.begin(), BuildVector.end());
5641 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5642 return true;
5643}
5644
5645/// \brief Like findBuildVector, but looks for construction of aggregate.
5646///
5647/// \return true if it matches.
5648static bool findBuildAggregate(InsertValueInst *IV,
5649 SmallVectorImpl<Value *> &BuildVector,
5650 SmallVectorImpl<Value *> &BuildVectorOpds) {
5651 Value *V;
5652 do {
5653 BuildVector.push_back(IV);
5654 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
5655 V = IV->getAggregateOperand();
5656 if (isa<UndefValue>(V))
5657 break;
5658 IV = dyn_cast<InsertValueInst>(V);
5659 if (!IV || !IV->hasOneUse())
5660 return false;
5661 } while (true);
5662 std::reverse(BuildVector.begin(), BuildVector.end());
5663 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5664 return true;
5665}
5666
5667static bool PhiTypeSorterFunc(Value *V, Value *V2) {
5668 return V->getType() < V2->getType();
5669}
5670
5671/// \brief Try and get a reduction value from a phi node.
5672///
5673/// Given a phi node \p P in a block \p ParentBB, consider possible reductions
5674/// if they come from either \p ParentBB or a containing loop latch.
5675///
5676/// \returns A candidate reduction value if possible, or \code nullptr \endcode
5677/// if not possible.
5678static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
5679 BasicBlock *ParentBB, LoopInfo *LI) {
5680 // There are situations where the reduction value is not dominated by the
5681 // reduction phi. Vectorizing such cases has been reported to cause
5682 // miscompiles. See PR25787.
5683 auto DominatedReduxValue = [&](Value *R) {
5684 return (
5685 dyn_cast<Instruction>(R) &&
5686 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
5687 };
5688
5689 Value *Rdx = nullptr;
5690
5691 // Return the incoming value if it comes from the same BB as the phi node.
5692 if (P->getIncomingBlock(0) == ParentBB) {
5693 Rdx = P->getIncomingValue(0);
5694 } else if (P->getIncomingBlock(1) == ParentBB) {
5695 Rdx = P->getIncomingValue(1);
5696 }
5697
5698 if (Rdx && DominatedReduxValue(Rdx))
5699 return Rdx;
5700
5701 // Otherwise, check whether we have a loop latch to look at.
5702 Loop *BBL = LI->getLoopFor(ParentBB);
5703 if (!BBL)
5704 return nullptr;
5705 BasicBlock *BBLatch = BBL->getLoopLatch();
5706 if (!BBLatch)
5707 return nullptr;
5708
5709 // There is a loop latch, return the incoming value if it comes from
5710 // that. This reduction pattern occasionally turns up.
5711 if (P->getIncomingBlock(0) == BBLatch) {
5712 Rdx = P->getIncomingValue(0);
5713 } else if (P->getIncomingBlock(1) == BBLatch) {
5714 Rdx = P->getIncomingValue(1);
5715 }
5716
5717 if (Rdx && DominatedReduxValue(Rdx))
5718 return Rdx;
5719
5720 return nullptr;
5721}
5722
5723/// Attempt to reduce a horizontal reduction.
5724/// If it is legal to match a horizontal reduction feeding the phi node \a P
5725/// with reduction operators \a Root (or one of its operands) in a basic block
5726/// \a BB, then check if it can be done. If horizontal reduction is not found
5727/// and root instruction is a binary operation, vectorization of the operands is
5728/// attempted.
5729/// \returns true if a horizontal reduction was matched and reduced or operands
5730/// of one of the binary instruction were vectorized.
5731/// \returns false if a horizontal reduction was not matched (or not possible)
5732/// or no vectorization of any binary operation feeding \a Root instruction was
5733/// performed.
5734static bool tryToVectorizeHorReductionOrInstOperands(
5735 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
5736 TargetTransformInfo *TTI,
5737 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
5738 if (!ShouldVectorizeHor)
5739 return false;
5740
5741 if (!Root)
5742 return false;
5743
5744 if (Root->getParent() != BB || isa<PHINode>(Root))
5745 return false;
5746 // Start analysis starting from Root instruction. If horizontal reduction is
5747 // found, try to vectorize it. If it is not a horizontal reduction or
5748 // vectorization is not possible or not effective, and currently analyzed
5749 // instruction is a binary operation, try to vectorize the operands, using
5750 // pre-order DFS traversal order. If the operands were not vectorized, repeat
5751 // the same procedure considering each operand as a possible root of the
5752 // horizontal reduction.
5753 // Interrupt the process if the Root instruction itself was vectorized or all
5754 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
5755 SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
5756 SmallSet<Value *, 8> VisitedInstrs;
5757 bool Res = false;
5758 while (!Stack.empty()) {
5759 Value *V;
5760 unsigned Level;
5761 std::tie(V, Level) = Stack.pop_back_val();
5762 if (!V)
5763 continue;
5764 auto *Inst = dyn_cast<Instruction>(V);
5765 if (!Inst)
5766 continue;
5767 auto *BI = dyn_cast<BinaryOperator>(Inst);
5768 auto *SI = dyn_cast<SelectInst>(Inst);
5769 if (BI || SI) {
5770 HorizontalReduction HorRdx;
5771 if (HorRdx.matchAssociativeReduction(P, Inst)) {
5772 if (HorRdx.tryToReduce(R, TTI)) {
5773 Res = true;
5774 // Set P to nullptr to avoid re-analysis of phi node in
5775 // matchAssociativeReduction function unless this is the root node.
5776 P = nullptr;
5777 continue;
5778 }
5779 }
5780 if (P && BI) {
5781 Inst = dyn_cast<Instruction>(BI->getOperand(0));
5782 if (Inst == P)
5783 Inst = dyn_cast<Instruction>(BI->getOperand(1));
5784 if (!Inst) {
5785 // Set P to nullptr to avoid re-analysis of phi node in
5786 // matchAssociativeReduction function unless this is the root node.
5787 P = nullptr;
5788 continue;
5789 }
5790 }
5791 }
5792 // Set P to nullptr to avoid re-analysis of phi node in
5793 // matchAssociativeReduction function unless this is the root node.
5794 P = nullptr;
5795 if (Vectorize(Inst, R)) {
5796 Res = true;
5797 continue;
5798 }
5799
5800 // Try to vectorize operands.
5801 // Continue analysis for the instruction from the same basic block only to
5802 // save compile time.
5803 if (++Level < RecursionMaxDepth)
5804 for (auto *Op : Inst->operand_values())
5805 if (VisitedInstrs.insert(Op).second)
5806 if (auto *I = dyn_cast<Instruction>(Op))
5807 if (!isa<PHINode>(I) && I->getParent() == BB)
5808 Stack.emplace_back(Op, Level);
5809 }
5810 return Res;
5811}
5812
5813bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
5814 BasicBlock *BB, BoUpSLP &R,
5815 TargetTransformInfo *TTI) {
5816 if (!V)
5817 return false;
5818 auto *I = dyn_cast<Instruction>(V);
5819 if (!I)
5820 return false;
5821
5822 if (!isa<BinaryOperator>(I))
5823 P = nullptr;
5824 // Try to match and vectorize a horizontal reduction.
5825 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
5826 return tryToVectorize(I, R);
5827 };
5828 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
5829 ExtraVectorization);
5830}
5831
5832bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
5833 BasicBlock *BB, BoUpSLP &R) {
5834 const DataLayout &DL = BB->getModule()->getDataLayout();
5835 if (!R.canMapToVector(IVI->getType(), DL))
5836 return false;
5837
5838 SmallVector<Value *, 16> BuildVector;
5839 SmallVector<Value *, 16> BuildVectorOpds;
5840 if (!findBuildAggregate(IVI, BuildVector, BuildVectorOpds))
5841 return false;
5842
5843 DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: array mappable to vector: " <<
*IVI << "\n"; } } while (false)
;
5844 return tryToVectorizeList(BuildVectorOpds, R, BuildVector, false);
5845}
5846
5847bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
5848 BasicBlock *BB, BoUpSLP &R) {
5849 SmallVector<Value *, 16> BuildVector;
5850 SmallVector<Value *, 16> BuildVectorOpds;
5851 if (!findBuildVector(IEI, BuildVector, BuildVectorOpds))
5852 return false;
5853
5854 // Vectorize starting with the build vector operands ignoring the BuildVector
5855 // instructions for the purpose of scheduling and user extraction.
5856 return tryToVectorizeList(BuildVectorOpds, R, BuildVector);
5857}
5858
5859bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
5860 BoUpSLP &R) {
5861 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
5862 return true;
5863
5864 bool OpsChanged = false;
5865 for (int Idx = 0; Idx < 2; ++Idx) {
5866 OpsChanged |=
5867 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
5868 }
5869 return OpsChanged;
5870}
5871
5872bool SLPVectorizerPass::vectorizeSimpleInstructions(
5873 SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
5874 bool OpsChanged = false;
5875 for (auto &VH : reverse(Instructions)) {
5876 auto *I = dyn_cast_or_null<Instruction>(VH);
5877 if (!I)
5878 continue;
5879 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
5880 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
5881 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
5882 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
5883 else if (auto *CI = dyn_cast<CmpInst>(I))
5884 OpsChanged |= vectorizeCmpInst(CI, BB, R);
5885 }
5886 Instructions.clear();
5887 return OpsChanged;
5888}
5889
5890bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
5891 bool Changed = false;
5892 SmallVector<Value *, 4> Incoming;
5893 SmallSet<Value *, 16> VisitedInstrs;
5894
5895 bool HaveVectorizedPhiNodes = true;
5896 while (HaveVectorizedPhiNodes) {
5897 HaveVectorizedPhiNodes = false;
5898
5899 // Collect the incoming values from the PHIs.
5900 Incoming.clear();
5901 for (Instruction &I : *BB) {
5902 PHINode *P = dyn_cast<PHINode>(&I);
5903 if (!P)
5904 break;
5905
5906 if (!VisitedInstrs.count(P))
5907 Incoming.push_back(P);
5908 }
5909
5910 // Sort by type.
5911 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
5912
5913 // Try to vectorize elements base on their type.
5914 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
5915 E = Incoming.end();
5916 IncIt != E;) {
5917
5918 // Look for the next elements with the same type.
5919 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
5920 while (SameTypeIt != E &&
5921 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
5922 VisitedInstrs.insert(*SameTypeIt);
5923 ++SameTypeIt;
5924 }
5925
5926 // Try to vectorize them.
5927 unsigned NumElts = (SameTypeIt - IncIt);
5928 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { errs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false)
;
5929 // The order in which the phi nodes appear in the program does not matter.
5930 // So allow tryToVectorizeList to reorder them if it is beneficial. This
5931 // is done when there are exactly two elements since tryToVectorizeList
5932 // asserts that there are only two values when AllowReorder is true.
5933 bool AllowReorder = NumElts == 2;
5934 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
5935 None, AllowReorder)) {
5936 // Success start over because instructions might have been changed.
5937 HaveVectorizedPhiNodes = true;
5938 Changed = true;
5939 break;
5940 }
5941
5942 // Start over at the next instruction of a different type (or the end).
5943 IncIt = SameTypeIt;
5944 }
5945 }
5946
5947 VisitedInstrs.clear();
5948
5949 SmallVector<WeakVH, 8> PostProcessInstructions;
5950 SmallDenseSet<Instruction *, 4> KeyNodes;
5951 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
5952 // We may go through BB multiple times so skip the one we have checked.
5953 if (!VisitedInstrs.insert(&*it).second) {
5954 if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
5955 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
5956 // We would like to start over since some instructions are deleted
5957 // and the iterator may become invalid value.
5958 Changed = true;
5959 it = BB->begin();
5960 e = BB->end();
5961 }
5962 continue;
5963 }
5964
5965 if (isa<DbgInfoIntrinsic>(it))
5966 continue;
5967
5968 // Try to vectorize reductions that use PHINodes.
5969 if (PHINode *P = dyn_cast<PHINode>(it)) {
5970 // Check that the PHI is a reduction PHI.
5971 if (P->getNumIncomingValues() != 2)
5972 return Changed;
5973
5974 // Try to match and vectorize a horizontal reduction.
5975 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
5976 TTI)) {
5977 Changed = true;
5978 it = BB->begin();
5979 e = BB->end();
5980 continue;
5981 }
5982 continue;
5983 }
5984
5985 // Ran into an instruction without users, like terminator, or function call
5986 // with ignored return value, store. Ignore unused instructions (basing on
5987 // instruction type, except for CallInst and InvokeInst).
5988 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
5989 isa<InvokeInst>(it))) {
5990 KeyNodes.insert(&*it);
5991 bool OpsChanged = false;
5992 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
5993 for (auto *V : it->operand_values()) {
5994 // Try to match and vectorize a horizontal reduction.
5995 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
5996 }
5997 }
5998 // Start vectorization of post-process list of instructions from the
5999 // top-tree instructions to try to vectorize as many instructions as
6000 // possible.
6001 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
6002 if (OpsChanged) {
6003 // We would like to start over since some instructions are deleted
6004 // and the iterator may become invalid value.
6005 Changed = true;
6006 it = BB->begin();
6007 e = BB->end();
6008 continue;
6009 }
6010 }
6011
6012 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
6013 isa<InsertValueInst>(it))
6014 PostProcessInstructions.push_back(&*it);
6015
6016 }
6017
6018 return Changed;
6019}
6020
6021bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
6022 auto Changed = false;
6023 for (auto &Entry : GEPs) {
6024 // If the getelementptr list has fewer than two elements, there's nothing
6025 // to do.
6026 if (Entry.second.size() < 2)
6027 continue;
6028
6029 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
6030 << Entry.second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
;
6031
6032 // We process the getelementptr list in chunks of 16 (like we do for
6033 // stores) to minimize compile-time.
6034 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
6035 auto Len = std::min<unsigned>(BE - BI, 16);
6036 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
6037
6038 // Initialize a set a candidate getelementptrs. Note that we use a
6039 // SetVector here to preserve program order. If the index computations
6040 // are vectorizable and begin with loads, we want to minimize the chance
6041 // of having to reorder them later.
6042 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
6043
6044 // Some of the candidates may have already been vectorized after we
6045 // initially collected them. If so, the WeakTrackingVHs will have
6046 // nullified the
6047 // values, so remove them from the set of candidates.
6048 Candidates.remove(nullptr);
6049
6050 // Remove from the set of candidates all pairs of getelementptrs with
6051 // constant differences. Such getelementptrs are likely not good
6052 // candidates for vectorization in a bottom-up phase since one can be
6053 // computed from the other. We also ensure all candidate getelementptr
6054 // indices are unique.
6055 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
6056 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
6057 if (!Candidates.count(GEPI))
6058 continue;
6059 auto *SCEVI = SE->getSCEV(GEPList[I]);
6060 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
6061 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
6062 auto *SCEVJ = SE->getSCEV(GEPList[J]);
6063 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
6064 Candidates.remove(GEPList[I]);
6065 Candidates.remove(GEPList[J]);
6066 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
6067 Candidates.remove(GEPList[J]);
6068 }
6069 }
6070 }
6071
6072 // We break out of the above computation as soon as we know there are
6073 // fewer than two candidates remaining.
6074 if (Candidates.size() < 2)
6075 continue;
6076
6077 // Add the single, non-constant index of each candidate to the bundle. We
6078 // ensured the indices met these constraints when we originally collected
6079 // the getelementptrs.
6080 SmallVector<Value *, 16> Bundle(Candidates.size());
6081 auto BundleIndex = 0u;
6082 for (auto *V : Candidates) {
6083 auto *GEP = cast<GetElementPtrInst>(V);
6084 auto *GEPIdx = GEP->idx_begin()->get();
6085 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx))((GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx
)) ? static_cast<void> (0) : __assert_fail ("GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)"
, "/build/llvm-toolchain-snapshot-6.0~svn318211/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 6085, __PRETTY_FUNCTION__))
;
6086 Bundle[BundleIndex++] = GEPIdx;
6087 }
6088
6089 // Try and vectorize the indices. We are currently only interested in
6090 // gather-like cases of the form:
6091 //
6092 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
6093 //
6094 // where the loads of "a", the loads of "b", and the subtractions can be
6095 // performed in parallel. It's likely that detecting this pattern in a
6096 // bottom-up phase will be simpler and less costly than building a
6097 // full-blown top-down phase beginning at the consecutive loads.
6098 Changed |= tryToVectorizeList(Bundle, R);
6099 }
6100 }
6101 return Changed;
6102}
6103
6104bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
6105 bool Changed = false;
6106 // Attempt to sort and vectorize each of the store-groups.
6107 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
6108 ++it) {
6109 if (it->second.size() < 2)
6110 continue;
6111
6112 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
6113 << it->second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
;
6114
6115 // Process the stores in chunks of 16.
6116 // TODO: The limit of 16 inhibits greater vectorization factors.
6117 // For example, AVX2 supports v32i8. Increasing this limit, however,
6118 // may cause a significant compile-time increase.
6119 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
6120 unsigned Len = std::min<unsigned>(CE - CI, 16);
6121 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
6122 }
6123 }
6124 return Changed;
6125}
6126
6127char SLPVectorizer::ID = 0;
6128
6129static const char lv_name[] = "SLP Vectorizer";
6130
6131INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)static void *initializeSLPVectorizerPassOnce(PassRegistry &
Registry) {
6132INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
6133INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
6134INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
6135INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
6136INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry);
6137INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
6138INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
6139INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "slp-vectorizer", &
SLPVectorizer::ID, PassInfo::NormalCtor_t(callDefaultCtor<
SLPVectorizer>), false, false); Registry.registerPass(*PI,
true); return PI; } static llvm::once_flag InitializeSLPVectorizerPassFlag
; void llvm::initializeSLPVectorizerPass(PassRegistry &Registry
) { llvm::call_once(InitializeSLPVectorizerPassFlag, initializeSLPVectorizerPassOnce
, std::ref(Registry)); }
6140
6141Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); }