Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 3062, column 27
Called C++ object pointer is null

Annotated Source Code

[?] Use j/k keys for keyboard navigation

/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp

→
1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11// stores that can be put together into vector-stores. Next, it attempts to
12// construct vectorizable tree using the use-def chains. If a profitable tree
13// was found, the SLP vectorizer performs vectorization on the tree.
14//
15// The pass is inspired by the work described in the paper:
16// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/DenseSet.h"
24#include "llvm/ADT/MapVector.h"
25#include "llvm/ADT/None.h"
26#include "llvm/ADT/Optional.h"
27#include "llvm/ADT/PostOrderIterator.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SetVector.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/iterator.h"
35#include "llvm/ADT/iterator_range.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/Analysis/CodeMetrics.h"
38#include "llvm/Analysis/DemandedBits.h"
39#include "llvm/Analysis/GlobalsModRef.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PassManager.h"
69#include "llvm/IR/PatternMatch.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/IR/Verifier.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/DOTGraphTraits.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/ErrorHandling.h"
83#include "llvm/Support/GraphWriter.h"
84#include "llvm/Support/KnownBits.h"
85#include "llvm/Support/MathExtras.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Transforms/Utils/LoopUtils.h"
88#include "llvm/Transforms/Vectorize.h"
89#include <algorithm>
90#include <cassert>
91#include <cstdint>
92#include <iterator>
93#include <memory>
94#include <set>
95#include <string>
96#include <tuple>
97#include <utility>
98#include <vector>
99
100using namespace llvm;
101using namespace llvm::PatternMatch;
102using namespace slpvectorizer;
103
104#define SV_NAME"slp-vectorizer" "slp-vectorizer"
105#define DEBUG_TYPE"SLP" "SLP"
106
107STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, false};
108
109static cl::opt<int>
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
112 "number "));
113
114static cl::opt<bool>
115ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
117
118static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120 cl::desc(
121 "Attempt to vectorize horizontal reductions feeding into a store"));
122
123static cl::opt<int>
124MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
126
127/// Limits the size of scheduling regions in a block.
128/// It avoid long compile times for _very_ large blocks where vector
129/// instructions are spread over a wide range.
130/// This limit is way higher than needed by real-world functions.
131static cl::opt<int>
132ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
134
135static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
138
139static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
142
143static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
146
147static cl::opt<bool>
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
150
151// Limit the number of alias checks. The limit is chosen so that
152// it has no negative effect on the llvm benchmarks.
153static const unsigned AliasedCheckLimit = 10;
154
155// Another limit for the alias checks: The maximum distance between load/store
156// instructions where alias checks are done.
157// This limit is useful for very large basic blocks.
158static const unsigned MaxMemDepDistance = 160;
159
160/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161/// regions to be handled.
162static const int MinScheduleRegionSize = 16;
163
164/// \brief Predicate for the element types that the SLP vectorizer supports.
165///
166/// The most important thing to filter here are types which are invalid in LLVM
167/// vectors. We also filter target specific types which have absolutely no
168/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169/// avoids spending time checking the cost model and realizing that they will
170/// be inevitably scalarized.
171static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
174}
175
176/// \returns true if all of the instructions in \p VL are in the same block or
177/// false otherwise.
178static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180 if (!I0)
181 return false;
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
185 if (!I)
186 return false;
187
188 if (BB != I->getParent())
189 return false;
190 }
191 return true;
192}
193
194/// \returns True if all of the values in \p VL are constants.
195static bool allConstant(ArrayRef<Value *> VL) {
196 for (Value *i : VL)
197 if (!isa<Constant>(i))
198 return false;
199 return true;
200}
201
202/// \returns True if all of the values in \p VL are identical.
203static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
205 if (VL[i] != VL[0])
206 return false;
207 return true;
208}
209
210/// Checks if the vector of instructions can be represented as a shuffle, like:
211/// %x0 = extractelement <4 x i8> %x, i32 0
212/// %x3 = extractelement <4 x i8> %x, i32 3
213/// %y1 = extractelement <4 x i8> %y, i32 1
214/// %y2 = extractelement <4 x i8> %y, i32 2
215/// %x0x0 = mul i8 %x0, %x0
216/// %x3x3 = mul i8 %x3, %x3
217/// %y1y1 = mul i8 %y1, %y1
218/// %y2y2 = mul i8 %y2, %y2
219/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223/// ret <4 x i8> %ins4
224/// can be transformed into:
225/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226/// i32 6>
227/// %2 = mul <4 x i8> %1, %1
228/// ret <4 x i8> %2
229/// We convert this initially to something like:
230/// %x0 = extractelement <4 x i8> %x, i32 0
231/// %x3 = extractelement <4 x i8> %x, i32 3
232/// %y1 = extractelement <4 x i8> %y, i32 1
233/// %y2 = extractelement <4 x i8> %y, i32 2
234/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238/// %5 = mul <4 x i8> %4, %4
239/// %6 = extractelement <4 x i8> %5, i32 0
240/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241/// %7 = extractelement <4 x i8> %5, i32 1
242/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243/// %8 = extractelement <4 x i8> %5, i32 2
244/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245/// %9 = extractelement <4 x i8> %5, i32 3
246/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247/// ret <4 x i8> %ins4
248/// InstCombiner transforms this into a shuffle and vector mul
249static Optional<TargetTransformInfo::ShuffleKind>
250isShuffle(ArrayRef<Value *> VL) {
251 auto *EI0 = cast<ExtractElementInst>(VL[0]);
252 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253 Value *Vec1 = nullptr;
254 Value *Vec2 = nullptr;
255 enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256 ShuffleMode CommonShuffleMode = Unknown;
257 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258 auto *EI = cast<ExtractElementInst>(VL[I]);
259 auto *Vec = EI->getVectorOperand();
260 // All vector operands must have the same number of vector elements.
261 if (Vec->getType()->getVectorNumElements() != Size)
262 return None;
263 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264 if (!Idx)
265 return None;
266 // Undefined behavior if Idx is negative or >= Size.
267 if (Idx->getValue().uge(Size))
268 continue;
269 unsigned IntIdx = Idx->getValue().getZExtValue();
270 // We can extractelement from undef vector.
271 if (isa<UndefValue>(Vec))
272 continue;
273 // For correct shuffling we have to have at most 2 different vector operands
274 // in all extractelement instructions.
275 if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276 return None;
277 if (CommonShuffleMode == Permute)
278 continue;
279 // If the extract index is not the same as the operation number, it is a
280 // permutation.
281 if (IntIdx != I) {
282 CommonShuffleMode = Permute;
283 continue;
284 }
285 // Check the shuffle mode for the current operation.
286 if (!Vec1)
287 Vec1 = Vec;
288 else if (Vec != Vec1)
289 Vec2 = Vec;
290 // Example: shufflevector A, B, <0,5,2,7>
291 // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292 // I is even and IntIdx for B == I - FirstAlternate shuffle.
293 // Example: shufflevector A, B, <4,1,6,3>
294 // I is even and IntIdx for A == I - SecondAlternate shuffle.
295 // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296 const bool IIsEven = I & 1;
297 const bool CurrVecIsA = Vec == Vec1;
298 const bool IIsOdd = !IIsEven;
299 const bool CurrVecIsB = !CurrVecIsA;
300 ShuffleMode CurrentShuffleMode =
301 ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302 : SecondAlternate;
303 // Common mode is not set or the same as the shuffle mode of the current
304 // operation - alternate.
305 if (CommonShuffleMode == Unknown)
306 CommonShuffleMode = CurrentShuffleMode;
307 // Common shuffle mode is not the same as the shuffle mode of the current
308 // operation - permutation.
309 if (CommonShuffleMode != CurrentShuffleMode)
310 CommonShuffleMode = Permute;
311 }
312 // If we're not crossing lanes in different vectors, consider it as blending.
313 if ((CommonShuffleMode == FirstAlternate ||
314 CommonShuffleMode == SecondAlternate) &&
315 Vec2)
316 return TargetTransformInfo::SK_Alternate;
317 // If Vec2 was never used, we have a permutation of a single vector, otherwise
318 // we have permutation of 2 vectors.
319 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
320 : TargetTransformInfo::SK_PermuteSingleSrc;
321}
322
323///\returns Opcode that can be clubbed with \p Op to create an alternate
324/// sequence which can later be merged as a ShuffleVector instruction.
325static unsigned getAltOpcode(unsigned Op) {
326 switch (Op) {
327 case Instruction::FAdd:
328 return Instruction::FSub;
329 case Instruction::FSub:
330 return Instruction::FAdd;
331 case Instruction::Add:
332 return Instruction::Sub;
333 case Instruction::Sub:
334 return Instruction::Add;
335 default:
336 return 0;
337 }
338}
339
340static bool isOdd(unsigned Value) {
341 return Value & 1;
342}
343
344static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345 unsigned CheckedOpcode) {
346 return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347}
348
349/// Chooses the correct key for scheduling data. If \p Op has the same (or
350/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
351/// OpValue.
352static Value *isOneOf(Value *OpValue, Value *Op) {
353 auto *I = dyn_cast<Instruction>(Op);
354 if (!I)
355 return OpValue;
356 auto *OpInst = cast<Instruction>(OpValue);
357 unsigned OpInstOpcode = OpInst->getOpcode();
358 unsigned IOpcode = I->getOpcode();
359 if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
360 return Op;
361 return OpValue;
362}
363
364namespace {
365
366/// Contains data for the instructions going to be vectorized.
367struct RawInstructionsData {
368 /// Main Opcode of the instructions going to be vectorized.
369 unsigned Opcode = 0;
370
371 /// The list of instructions have some instructions with alternate opcodes.
372 bool HasAltOpcodes = false;
373};
374
375} // end anonymous namespace
376
377/// Checks the list of the vectorized instructions \p VL and returns info about
378/// this list.
379static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380 auto *I0 = dyn_cast<Instruction>(VL[0]);
381 if (!I0)
382 return {};
383 RawInstructionsData Res;
384 unsigned Opcode = I0->getOpcode();
385 // Walk through the list of the vectorized instructions
386 // in order to check its structure described by RawInstructionsData.
387 for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388 auto *I = dyn_cast<Instruction>(VL[Cnt]);
389 if (!I)
390 return {};
391 if (Opcode != I->getOpcode())
392 Res.HasAltOpcodes = true;
393 }
394 Res.Opcode = Opcode;
395 return Res;
396}
397
398namespace {
399
400/// Main data required for vectorization of instructions.
401struct InstructionsState {
402 /// The very first instruction in the list with the main opcode.
403 Value *OpValue = nullptr;
404
405 /// The main opcode for the list of instructions.
406 unsigned Opcode = 0;
407
408 /// Some of the instructions in the list have alternate opcodes.
409 bool IsAltShuffle = false;
410
411 InstructionsState() = default;
412 InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413 : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
414};
415
416} // end anonymous namespace
417
418/// \returns analysis of the Instructions in \p VL described in
419/// InstructionsState, the Opcode that we suppose the whole list
420/// could be vectorized even if its structure is diverse.
421static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422 auto Res = getMainOpcode(VL);
423 unsigned Opcode = Res.Opcode;
424 if (!Res.HasAltOpcodes)
425 return InstructionsState(VL[0], Opcode, false);
426 auto *OpInst = cast<Instruction>(VL[0]);
427 unsigned AltOpcode = getAltOpcode(Opcode);
428 // Examine each element in the list instructions VL to determine
429 // if some operations there could be considered as an alternative
430 // (for example as subtraction relates to addition operation).
431 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432 auto *I = cast<Instruction>(VL[Cnt]);
433 unsigned InstOpcode = I->getOpcode();
434 if ((Res.HasAltOpcodes &&
435 InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436 (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437 return InstructionsState(OpInst, 0, false);
438 }
439 }
440 return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
441}
442
443/// \returns true if all of the values in \p VL have the same type or false
444/// otherwise.
445static bool allSameType(ArrayRef<Value *> VL) {
446 Type *Ty = VL[0]->getType();
447 for (int i = 1, e = VL.size(); i < e; i++)
448 if (VL[i]->getType() != Ty)
449 return false;
450
451 return true;
452}
453
454/// \returns True if Extract{Value,Element} instruction extracts element Idx.
455static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
456 assert(Opcode == Instruction::ExtractElement ||(static_cast <bool> (Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) ? void (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 457, __extension__ __PRETTY_FUNCTION__))
457 Opcode == Instruction::ExtractValue)(static_cast <bool> (Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) ? void (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 457, __extension__ __PRETTY_FUNCTION__));
458 if (Opcode == Instruction::ExtractElement) {
459 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
460 return CI && CI->getZExtValue() == Idx;
461 } else {
462 ExtractValueInst *EI = cast<ExtractValueInst>(E);
463 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
464 }
465}
466
467/// \returns True if in-tree use also needs extract. This refers to
468/// possible scalar operand in vectorized instruction.
469static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
470 TargetLibraryInfo *TLI) {
471 unsigned Opcode = UserInst->getOpcode();
472 switch (Opcode) {
473 case Instruction::Load: {
474 LoadInst *LI = cast<LoadInst>(UserInst);
475 return (LI->getPointerOperand() == Scalar);
476 }
477 case Instruction::Store: {
478 StoreInst *SI = cast<StoreInst>(UserInst);
479 return (SI->getPointerOperand() == Scalar);
480 }
481 case Instruction::Call: {
482 CallInst *CI = cast<CallInst>(UserInst);
483 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
484 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
485 return (CI->getArgOperand(1) == Scalar);
486 }
487 LLVM_FALLTHROUGH[[clang::fallthrough]];
488 }
489 default:
490 return false;
491 }
492}
493
494/// \returns the AA location that is being access by the instruction.
495static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
496 if (StoreInst *SI = dyn_cast<StoreInst>(I))
497 return MemoryLocation::get(SI);
498 if (LoadInst *LI = dyn_cast<LoadInst>(I))
499 return MemoryLocation::get(LI);
500 return MemoryLocation();
501}
502
503/// \returns True if the instruction is not a volatile or atomic load/store.
504static bool isSimple(Instruction *I) {
505 if (LoadInst *LI = dyn_cast<LoadInst>(I))
506 return LI->isSimple();
507 if (StoreInst *SI = dyn_cast<StoreInst>(I))
508 return SI->isSimple();
509 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
510 return !MI->isVolatile();
511 return true;
512}
513
514namespace llvm {
515
516namespace slpvectorizer {
517
518/// Bottom Up SLP Vectorizer.
519class BoUpSLP {
520public:
521 using ValueList = SmallVector<Value *, 8>;
522 using InstrList = SmallVector<Instruction *, 16>;
523 using ValueSet = SmallPtrSet<Value *, 16>;
524 using StoreList = SmallVector<StoreInst *, 8>;
525 using ExtraValueToDebugLocsMap =
526 MapVector<Value *, SmallVector<Instruction *, 2>>;
527
528 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
529 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
530 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
531 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
532 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
533 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
534 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
535 // Use the vector register size specified by the target unless overridden
536 // by a command-line option.
537 // TODO: It would be better to limit the vectorization factor based on
538 // data type rather than just register size. For example, x86 AVX has
539 // 256-bit registers, but it does not support integer operations
540 // at that width (that requires AVX2).
541 if (MaxVectorRegSizeOption.getNumOccurrences())
542 MaxVecRegSize = MaxVectorRegSizeOption;
543 else
544 MaxVecRegSize = TTI->getRegisterBitWidth(true);
545
546 if (MinVectorRegSizeOption.getNumOccurrences())
547 MinVecRegSize = MinVectorRegSizeOption;
548 else
549 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
550 }
551
552 /// \brief Vectorize the tree that starts with the elements in \p VL.
553 /// Returns the vectorized root.
554 Value *vectorizeTree();
555
556 /// Vectorize the tree but with the list of externally used values \p
557 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
558 /// generated extractvalue instructions.
559 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
560
561 /// \returns the cost incurred by unwanted spills and fills, caused by
562 /// holding live values over call sites.
563 int getSpillCost();
564
565 /// \returns the vectorization cost of the subtree that starts at \p VL.
566 /// A negative number means that this is profitable.
567 int getTreeCost();
568
569 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
570 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
571 void buildTree(ArrayRef<Value *> Roots,
572 ArrayRef<Value *> UserIgnoreLst = None);
573
574 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
576 /// into account (anf updating it, if required) list of externally used
577 /// values stored in \p ExternallyUsedValues.
578 void buildTree(ArrayRef<Value *> Roots,
579 ExtraValueToDebugLocsMap &ExternallyUsedValues,
580 ArrayRef<Value *> UserIgnoreLst = None);
581
582 /// Clear the internal data structures that are created by 'buildTree'.
583 void deleteTree() {
584 VectorizableTree.clear();
585 ScalarToTreeEntry.clear();
586 MustGather.clear();
587 ExternalUses.clear();
588 NumLoadsWantToKeepOrder = 0;
589 NumLoadsWantToChangeOrder = 0;
590 for (auto &Iter : BlocksSchedules) {
591 BlockScheduling *BS = Iter.second.get();
592 BS->clear();
593 }
594 MinBWs.clear();
595 }
596
597 unsigned getTreeSize() const { return VectorizableTree.size(); }
598
599 /// \brief Perform LICM and CSE on the newly generated gather sequences.
600 void optimizeGatherSequence(Function &F);
601
602 /// \returns true if it is beneficial to reverse the vector order.
603 bool shouldReorder() const {
604 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
605 }
606
607 /// \return The vector element size in bits to use when vectorizing the
608 /// expression tree ending at \p V. If V is a store, the size is the width of
609 /// the stored value. Otherwise, the size is the width of the largest loaded
610 /// value reaching V. This method is used by the vectorizer to calculate
611 /// vectorization factors.
612 unsigned getVectorElementSize(Value *V);
613
614 /// Compute the minimum type sizes required to represent the entries in a
615 /// vectorizable tree.
616 void computeMinimumValueSizes();
617
618 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
619 unsigned getMaxVecRegSize() const {
620 return MaxVecRegSize;
621 }
622
623 // \returns minimum vector register size as set by cl::opt.
624 unsigned getMinVecRegSize() const {
625 return MinVecRegSize;
626 }
627
628 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
629 ///
630 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
631 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
632
633 /// \returns True if the VectorizableTree is both tiny and not fully
634 /// vectorizable. We do not vectorize such trees.
635 bool isTreeTinyAndNotFullyVectorizable();
636
637 OptimizationRemarkEmitter *getORE() { return ORE; }
638
639private:
640 struct TreeEntry;
641
642 /// Checks if all users of \p I are the part of the vectorization tree.
643 bool areAllUsersVectorized(Instruction *I) const;
644
645 /// \returns the cost of the vectorizable entry.
646 int getEntryCost(TreeEntry *E);
647
648 /// This is the recursive part of buildTree.
649 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
650
651 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
652 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
653 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const;
654
655 /// Vectorize a single entry in the tree.
656 Value *vectorizeTree(TreeEntry *E);
657
658 /// Vectorize a single entry in the tree, starting in \p VL.
659 Value *vectorizeTree(ArrayRef<Value *> VL);
660
661 /// \returns the pointer to the vectorized value if \p VL is already
662 /// vectorized, or NULL. They may happen in cycles.
663 Value *alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const;
664
665 /// \returns the scalarization cost for this type. Scalarization in this
666 /// context means the creation of vectors from a group of scalars.
667 int getGatherCost(Type *Ty);
668
669 /// \returns the scalarization cost for this list of values. Assuming that
670 /// this subtree gets vectorized, we may need to extract the values from the
671 /// roots. This method calculates the cost of extracting the values.
672 int getGatherCost(ArrayRef<Value *> VL);
673
674 /// \brief Set the Builder insert point to one after the last instruction in
675 /// the bundle
676 void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
677
678 /// \returns a vector from a collection of scalars in \p VL.
679 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
680
681 /// \returns whether the VectorizableTree is fully vectorizable and will
682 /// be beneficial even the tree height is tiny.
683 bool isFullyVectorizableTinyTree();
684
685 /// \reorder commutative operands in alt shuffle if they result in
686 /// vectorized code.
687 void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
688 SmallVectorImpl<Value *> &Left,
689 SmallVectorImpl<Value *> &Right);
690
691 /// \reorder commutative operands to get better probability of
692 /// generating vectorized code.
693 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
694 SmallVectorImpl<Value *> &Left,
695 SmallVectorImpl<Value *> &Right);
696 struct TreeEntry {
697 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
698
699 /// \returns true if the scalars in VL are equal to this entry.
700 bool isSame(ArrayRef<Value *> VL) const {
701 assert(VL.size() == Scalars.size() && "Invalid size")(static_cast <bool> (VL.size() == Scalars.size() &&
"Invalid size") ? void (0) : __assert_fail ("VL.size() == Scalars.size() && \"Invalid size\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 701, __extension__ __PRETTY_FUNCTION__));
702 return std::equal(VL.begin(), VL.end(), Scalars.begin());
703 }
704
705 /// A vector of scalars.
706 ValueList Scalars;
707
708 /// The Scalars are vectorized into this value. It is initialized to Null.
709 Value *VectorizedValue = nullptr;
710
711 /// Do we need to gather this sequence ?
712 bool NeedToGather = false;
713
714 /// Points back to the VectorizableTree.
715 ///
716 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
717 /// to be a pointer and needs to be able to initialize the child iterator.
718 /// Thus we need a reference back to the container to translate the indices
719 /// to entries.
720 std::vector<TreeEntry> &Container;
721
722 /// The TreeEntry index containing the user of this entry. We can actually
723 /// have multiple users so the data structure is not truly a tree.
724 SmallVector<int, 1> UserTreeIndices;
725 };
726
727 /// Create a new VectorizableTree entry.
728 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
729 int &UserTreeIdx) {
730 VectorizableTree.emplace_back(VectorizableTree);
731 int idx = VectorizableTree.size() - 1;
732 TreeEntry *Last = &VectorizableTree[idx];
733 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
734 Last->NeedToGather = !Vectorized;
735 if (Vectorized) {
736 for (int i = 0, e = VL.size(); i != e; ++i) {
737 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!")(static_cast <bool> (!getTreeEntry(VL[i]) && "Scalar already in tree!"
) ? void (0) : __assert_fail ("!getTreeEntry(VL[i]) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 737, __extension__ __PRETTY_FUNCTION__));
738 ScalarToTreeEntry[VL[i]] = idx;
739 }
740 } else {
741 MustGather.insert(VL.begin(), VL.end());
742 }
743
744 if (UserTreeIdx >= 0)
745 Last->UserTreeIndices.push_back(UserTreeIdx);
746 UserTreeIdx = idx;
747 return Last;
748 }
749
750 /// -- Vectorization State --
751 /// Holds all of the tree entries.
752 std::vector<TreeEntry> VectorizableTree;
753
754 TreeEntry *getTreeEntry(Value *V) {
755 auto I = ScalarToTreeEntry.find(V);
756 if (I != ScalarToTreeEntry.end())
757 return &VectorizableTree[I->second];
758 return nullptr;
759 }
760
761 const TreeEntry *getTreeEntry(Value *V) const {
762 auto I = ScalarToTreeEntry.find(V);
763 if (I != ScalarToTreeEntry.end())
764 return &VectorizableTree[I->second];
765 return nullptr;
766 }
767
768 /// Maps a specific scalar to its tree entry.
769 SmallDenseMap<Value*, int> ScalarToTreeEntry;
770
771 /// A list of scalars that we found that we need to keep as scalars.
772 ValueSet MustGather;
773
774 /// This POD struct describes one external user in the vectorized tree.
775 struct ExternalUser {
776 ExternalUser(Value *S, llvm::User *U, int L)
777 : Scalar(S), User(U), Lane(L) {}
778
779 // Which scalar in our function.
780 Value *Scalar;
781
782 // Which user that uses the scalar.
783 llvm::User *User;
784
785 // Which lane does the scalar belong to.
786 int Lane;
787 };
788 using UserList = SmallVector<ExternalUser, 16>;
789
790 /// Checks if two instructions may access the same memory.
791 ///
792 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
793 /// is invariant in the calling loop.
794 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
795 Instruction *Inst2) {
796 // First check if the result is already in the cache.
797 AliasCacheKey key = std::make_pair(Inst1, Inst2);
798 Optional<bool> &result = AliasCache[key];
799 if (result.hasValue()) {
800 return result.getValue();
801 }
802 MemoryLocation Loc2 = getLocation(Inst2, AA);
803 bool aliased = true;
804 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
805 // Do the alias check.
806 aliased = AA->alias(Loc1, Loc2);
807 }
808 // Store the result in the cache.
809 result = aliased;
810 return aliased;
811 }
812
813 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
814
815 /// Cache for alias results.
816 /// TODO: consider moving this to the AliasAnalysis itself.
817 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
818
819 /// Removes an instruction from its block and eventually deletes it.
820 /// It's like Instruction::eraseFromParent() except that the actual deletion
821 /// is delayed until BoUpSLP is destructed.
822 /// This is required to ensure that there are no incorrect collisions in the
823 /// AliasCache, which can happen if a new instruction is allocated at the
824 /// same address as a previously deleted instruction.
825 void eraseInstruction(Instruction *I) {
826 I->removeFromParent();
827 I->dropAllReferences();
828 DeletedInstructions.emplace_back(I);
829 }
830
831 /// Temporary store for deleted instructions. Instructions will be deleted
832 /// eventually when the BoUpSLP is destructed.
833 SmallVector<unique_value, 8> DeletedInstructions;
834
835 /// A list of values that need to extracted out of the tree.
836 /// This list holds pairs of (Internal Scalar : External User). External User
837 /// can be nullptr, it means that this Internal Scalar will be used later,
838 /// after vectorization.
839 UserList ExternalUses;
840
841 /// Values used only by @llvm.assume calls.
842 SmallPtrSet<const Value *, 32> EphValues;
843
844 /// Holds all of the instructions that we gathered.
845 SetVector<Instruction *> GatherSeq;
846
847 /// A list of blocks that we are going to CSE.
848 SetVector<BasicBlock *> CSEBlocks;
849
850 /// Contains all scheduling relevant data for an instruction.
851 /// A ScheduleData either represents a single instruction or a member of an
852 /// instruction bundle (= a group of instructions which is combined into a
853 /// vector instruction).
854 struct ScheduleData {
855 // The initial value for the dependency counters. It means that the
856 // dependencies are not calculated yet.
857 enum { InvalidDeps = -1 };
858
859 ScheduleData() = default;
860
861 void init(int BlockSchedulingRegionID, Value *OpVal) {
862 FirstInBundle = this;
863 NextInBundle = nullptr;
864 NextLoadStore = nullptr;
865 IsScheduled = false;
866 SchedulingRegionID = BlockSchedulingRegionID;
867 UnscheduledDepsInBundle = UnscheduledDeps;
868 clearDependencies();
869 OpValue = OpVal;
870 }
871
872 /// Returns true if the dependency information has been calculated.
873 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
874
875 /// Returns true for single instructions and for bundle representatives
876 /// (= the head of a bundle).
877 bool isSchedulingEntity() const { return FirstInBundle == this; }
878
879 /// Returns true if it represents an instruction bundle and not only a
880 /// single instruction.
881 bool isPartOfBundle() const {
882 return NextInBundle != nullptr || FirstInBundle != this;
883 }
884
885 /// Returns true if it is ready for scheduling, i.e. it has no more
886 /// unscheduled depending instructions/bundles.
887 bool isReady() const {
888 assert(isSchedulingEntity() &&(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 889, __extension__ __PRETTY_FUNCTION__))
889 "can't consider non-scheduling entity for ready list")(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 889, __extension__ __PRETTY_FUNCTION__));
890 return UnscheduledDepsInBundle == 0 && !IsScheduled;
891 }
892
893 /// Modifies the number of unscheduled dependencies, also updating it for
894 /// the whole bundle.
895 int incrementUnscheduledDeps(int Incr) {
896 UnscheduledDeps += Incr;
897 return FirstInBundle->UnscheduledDepsInBundle += Incr;
898 }
899
900 /// Sets the number of unscheduled dependencies to the number of
901 /// dependencies.
902 void resetUnscheduledDeps() {
903 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
904 }
905
906 /// Clears all dependency information.
907 void clearDependencies() {
908 Dependencies = InvalidDeps;
909 resetUnscheduledDeps();
910 MemoryDependencies.clear();
911 }
912
913 void dump(raw_ostream &os) const {
914 if (!isSchedulingEntity()) {
915 os << "/ " << *Inst;
916 } else if (NextInBundle) {
917 os << '[' << *Inst;
918 ScheduleData *SD = NextInBundle;
919 while (SD) {
920 os << ';' << *SD->Inst;
921 SD = SD->NextInBundle;
922 }
923 os << ']';
924 } else {
925 os << *Inst;
926 }
927 }
928
929 Instruction *Inst = nullptr;
930
931 /// Points to the head in an instruction bundle (and always to this for
932 /// single instructions).
933 ScheduleData *FirstInBundle = nullptr;
934
935 /// Single linked list of all instructions in a bundle. Null if it is a
936 /// single instruction.
937 ScheduleData *NextInBundle = nullptr;
938
939 /// Single linked list of all memory instructions (e.g. load, store, call)
940 /// in the block - until the end of the scheduling region.
941 ScheduleData *NextLoadStore = nullptr;
942
943 /// The dependent memory instructions.
944 /// This list is derived on demand in calculateDependencies().
945 SmallVector<ScheduleData *, 4> MemoryDependencies;
946
947 /// This ScheduleData is in the current scheduling region if this matches
948 /// the current SchedulingRegionID of BlockScheduling.
949 int SchedulingRegionID = 0;
950
951 /// Used for getting a "good" final ordering of instructions.
952 int SchedulingPriority = 0;
953
954 /// The number of dependencies. Constitutes of the number of users of the
955 /// instruction plus the number of dependent memory instructions (if any).
956 /// This value is calculated on demand.
957 /// If InvalidDeps, the number of dependencies is not calculated yet.
958 int Dependencies = InvalidDeps;
959
960 /// The number of dependencies minus the number of dependencies of scheduled
961 /// instructions. As soon as this is zero, the instruction/bundle gets ready
962 /// for scheduling.
963 /// Note that this is negative as long as Dependencies is not calculated.
964 int UnscheduledDeps = InvalidDeps;
965
966 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
967 /// single instructions.
968 int UnscheduledDepsInBundle = InvalidDeps;
969
970 /// True if this instruction is scheduled (or considered as scheduled in the
971 /// dry-run).
972 bool IsScheduled = false;
973
974 /// Opcode of the current instruction in the schedule data.
975 Value *OpValue = nullptr;
976 };
977
978#ifndef NDEBUG
979 friend inline raw_ostream &operator<<(raw_ostream &os,
980 const BoUpSLP::ScheduleData &SD) {
981 SD.dump(os);
982 return os;
983 }
984#endif
985
986 friend struct GraphTraits<BoUpSLP *>;
987 friend struct DOTGraphTraits<BoUpSLP *>;
988
989 /// Contains all scheduling data for a basic block.
990 struct BlockScheduling {
991 BlockScheduling(BasicBlock *BB)
992 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
993
994 void clear() {
995 ReadyInsts.clear();
996 ScheduleStart = nullptr;
997 ScheduleEnd = nullptr;
998 FirstLoadStoreInRegion = nullptr;
999 LastLoadStoreInRegion = nullptr;
1000
1001 // Reduce the maximum schedule region size by the size of the
1002 // previous scheduling run.
1003 ScheduleRegionSizeLimit -= ScheduleRegionSize;
1004 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1005 ScheduleRegionSizeLimit = MinScheduleRegionSize;
1006 ScheduleRegionSize = 0;
1007
1008 // Make a new scheduling region, i.e. all existing ScheduleData is not
1009 // in the new region yet.
1010 ++SchedulingRegionID;
1011 }
1012
1013 ScheduleData *getScheduleData(Value *V) {
1014 ScheduleData *SD = ScheduleDataMap[V];
1015 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1016 return SD;
1017 return nullptr;
1018 }
1019
1020 ScheduleData *getScheduleData(Value *V, Value *Key) {
1021 if (V == Key)
1022 return getScheduleData(V);
1023 auto I = ExtraScheduleDataMap.find(V);
1024 if (I != ExtraScheduleDataMap.end()) {
1025 ScheduleData *SD = I->second[Key];
1026 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1027 return SD;
1028 }
1029 return nullptr;
1030 }
1031
1032 bool isInSchedulingRegion(ScheduleData *SD) {
1033 return SD->SchedulingRegionID == SchedulingRegionID;
1034 }
1035
1036 /// Marks an instruction as scheduled and puts all dependent ready
1037 /// instructions into the ready-list.
1038 template <typename ReadyListType>
1039 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1040 SD->IsScheduled = true;
1041 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false);
1042
1043 ScheduleData *BundleMember = SD;
1044 while (BundleMember) {
1045 if (BundleMember->Inst != BundleMember->OpValue) {
1046 BundleMember = BundleMember->NextInBundle;
1047 continue;
1048 }
1049 // Handle the def-use chain dependencies.
1050 for (Use &U : BundleMember->Inst->operands()) {
1051 auto *I = dyn_cast<Instruction>(U.get());
1052 if (!I)
1053 continue;
1054 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1055 if (OpDef && OpDef->hasValidDependencies() &&
1056 OpDef->incrementUnscheduledDeps(-1) == 0) {
1057 // There are no more unscheduled dependencies after
1058 // decrementing, so we can put the dependent instruction
1059 // into the ready list.
1060 ScheduleData *DepBundle = OpDef->FirstInBundle;
1061 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1062, __extension__ __PRETTY_FUNCTION__))
1062 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1062, __extension__ __PRETTY_FUNCTION__));
1063 ReadyList.insert(DepBundle);
1064 DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
1065 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false);
1066 }
1067 });
1068 }
1069 // Handle the memory dependencies.
1070 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1071 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1072 // There are no more unscheduled dependencies after decrementing,
1073 // so we can put the dependent instruction into the ready list.
1074 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1075 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1076, __extension__ __PRETTY_FUNCTION__))
1076 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1076, __extension__ __PRETTY_FUNCTION__));
1077 ReadyList.insert(DepBundle);
1078 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundledo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
1079 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false);
1080 }
1081 }
1082 BundleMember = BundleMember->NextInBundle;
1083 }
1084 }
1085
1086 void doForAllOpcodes(Value *V,
1087 function_ref<void(ScheduleData *SD)> Action) {
1088 if (ScheduleData *SD = getScheduleData(V))
1089 Action(SD);
1090 auto I = ExtraScheduleDataMap.find(V);
1091 if (I != ExtraScheduleDataMap.end())
1092 for (auto &P : I->second)
1093 if (P.second->SchedulingRegionID == SchedulingRegionID)
1094 Action(P.second);
1095 }
1096
1097 /// Put all instructions into the ReadyList which are ready for scheduling.
1098 template <typename ReadyListType>
1099 void initialFillReadyList(ReadyListType &ReadyList) {
1100 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1101 doForAllOpcodes(I, [&](ScheduleData *SD) {
1102 if (SD->isSchedulingEntity() && SD->isReady()) {
1103 ReadyList.insert(SD);
1104 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false);
1105 }
1106 });
1107 }
1108 }
1109
1110 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1111 /// cyclic dependencies. This is only a dry-run, no instructions are
1112 /// actually moved at this stage.
1113 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1114
1115 /// Un-bundles a group of instructions.
1116 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1117
1118 /// Allocates schedule data chunk.
1119 ScheduleData *allocateScheduleDataChunks();
1120
1121 /// Extends the scheduling region so that V is inside the region.
1122 /// \returns true if the region size is within the limit.
1123 bool extendSchedulingRegion(Value *V, Value *OpValue);
1124
1125 /// Initialize the ScheduleData structures for new instructions in the
1126 /// scheduling region.
1127 void initScheduleData(Instruction *FromI, Instruction *ToI,
1128 ScheduleData *PrevLoadStore,
1129 ScheduleData *NextLoadStore);
1130
1131 /// Updates the dependency information of a bundle and of all instructions/
1132 /// bundles which depend on the original bundle.
1133 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1134 BoUpSLP *SLP);
1135
1136 /// Sets all instruction in the scheduling region to un-scheduled.
1137 void resetSchedule();
1138
1139 BasicBlock *BB;
1140
1141 /// Simple memory allocation for ScheduleData.
1142 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1143
1144 /// The size of a ScheduleData array in ScheduleDataChunks.
1145 int ChunkSize;
1146
1147 /// The allocator position in the current chunk, which is the last entry
1148 /// of ScheduleDataChunks.
1149 int ChunkPos;
1150
1151 /// Attaches ScheduleData to Instruction.
1152 /// Note that the mapping survives during all vectorization iterations, i.e.
1153 /// ScheduleData structures are recycled.
1154 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1155
1156 /// Attaches ScheduleData to Instruction with the leading key.
1157 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1158 ExtraScheduleDataMap;
1159
1160 struct ReadyList : SmallVector<ScheduleData *, 8> {
1161 void insert(ScheduleData *SD) { push_back(SD); }
1162 };
1163
1164 /// The ready-list for scheduling (only used for the dry-run).
1165 ReadyList ReadyInsts;
1166
1167 /// The first instruction of the scheduling region.
1168 Instruction *ScheduleStart = nullptr;
1169
1170 /// The first instruction _after_ the scheduling region.
1171 Instruction *ScheduleEnd = nullptr;
1172
1173 /// The first memory accessing instruction in the scheduling region
1174 /// (can be null).
1175 ScheduleData *FirstLoadStoreInRegion = nullptr;
1176
1177 /// The last memory accessing instruction in the scheduling region
1178 /// (can be null).
1179 ScheduleData *LastLoadStoreInRegion = nullptr;
1180
1181 /// The current size of the scheduling region.
1182 int ScheduleRegionSize = 0;
1183
1184 /// The maximum size allowed for the scheduling region.
1185 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1186
1187 /// The ID of the scheduling region. For a new vectorization iteration this
1188 /// is incremented which "removes" all ScheduleData from the region.
1189 // Make sure that the initial SchedulingRegionID is greater than the
1190 // initial SchedulingRegionID in ScheduleData (which is 0).
1191 int SchedulingRegionID = 1;
1192 };
1193
1194 /// Attaches the BlockScheduling structures to basic blocks.
1195 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1196
1197 /// Performs the "real" scheduling. Done before vectorization is actually
1198 /// performed in a basic block.
1199 void scheduleBlock(BlockScheduling *BS);
1200
1201 /// List of users to ignore during scheduling and that don't need extracting.
1202 ArrayRef<Value *> UserIgnoreList;
1203
1204 // Number of load bundles that contain consecutive loads.
1205 int NumLoadsWantToKeepOrder = 0;
1206
1207 // Number of load bundles that contain consecutive loads in reversed order.
1208 int NumLoadsWantToChangeOrder = 0;
1209
1210 // Analysis and block reference.
1211 Function *F;
1212 ScalarEvolution *SE;
1213 TargetTransformInfo *TTI;
1214 TargetLibraryInfo *TLI;
1215 AliasAnalysis *AA;
1216 LoopInfo *LI;
1217 DominatorTree *DT;
1218 AssumptionCache *AC;
1219 DemandedBits *DB;
1220 const DataLayout *DL;
1221 OptimizationRemarkEmitter *ORE;
1222
1223 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1224 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1225
1226 /// Instruction builder to construct the vectorized tree.
1227 IRBuilder<> Builder;
1228
1229 /// A map of scalar integer values to the smallest bit width with which they
1230 /// can legally be represented. The values map to (width, signed) pairs,
1231 /// where "width" indicates the minimum bit width and "signed" is True if the
1232 /// value must be signed-extended, rather than zero-extended, back to its
1233 /// original width.
1234 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1235};
1236
1237} // end namespace slpvectorizer
1238
1239template <> struct GraphTraits<BoUpSLP *> {
1240 using TreeEntry = BoUpSLP::TreeEntry;
1241
1242 /// NodeRef has to be a pointer per the GraphWriter.
1243 using NodeRef = TreeEntry *;
1244
1245 /// \brief Add the VectorizableTree to the index iterator to be able to return
1246 /// TreeEntry pointers.
1247 struct ChildIteratorType
1248 : public iterator_adaptor_base<ChildIteratorType,
1249 SmallVector<int, 1>::iterator> {
1250 std::vector<TreeEntry> &VectorizableTree;
1251
1252 ChildIteratorType(SmallVector<int, 1>::iterator W,
1253 std::vector<TreeEntry> &VT)
1254 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1255
1256 NodeRef operator*() { return &VectorizableTree[*I]; }
1257 };
1258
1259 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1260
1261 static ChildIteratorType child_begin(NodeRef N) {
1262 return {N->UserTreeIndices.begin(), N->Container};
1263 }
1264
1265 static ChildIteratorType child_end(NodeRef N) {
1266 return {N->UserTreeIndices.end(), N->Container};
1267 }
1268
1269 /// For the node iterator we just need to turn the TreeEntry iterator into a
1270 /// TreeEntry* iterator so that it dereferences to NodeRef.
1271 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1272
1273 static nodes_iterator nodes_begin(BoUpSLP *R) {
1274 return nodes_iterator(R->VectorizableTree.begin());
1275 }
1276
1277 static nodes_iterator nodes_end(BoUpSLP *R) {
1278 return nodes_iterator(R->VectorizableTree.end());
1279 }
1280
1281 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1282};
1283
1284template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1285 using TreeEntry = BoUpSLP::TreeEntry;
1286
1287 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1288
1289 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1290 std::string Str;
1291 raw_string_ostream OS(Str);
1292 if (isSplat(Entry->Scalars)) {
1293 OS << "<splat> " << *Entry->Scalars[0];
1294 return Str;
1295 }
1296 for (auto V : Entry->Scalars) {
1297 OS << *V;
1298 if (std::any_of(
1299 R->ExternalUses.begin(), R->ExternalUses.end(),
1300 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1301 OS << " <extract>";
1302 OS << "\n";
1303 }
1304 return Str;
1305 }
1306
1307 static std::string getNodeAttributes(const TreeEntry *Entry,
1308 const BoUpSLP *) {
1309 if (Entry->NeedToGather)
1310 return "color=red";
1311 return "";
1312 }
1313};
1314
1315} // end namespace llvm
1316
1317void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1318 ArrayRef<Value *> UserIgnoreLst) {
1319 ExtraValueToDebugLocsMap ExternallyUsedValues;
1320 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1321}
1322
1323void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1324 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1325 ArrayRef<Value *> UserIgnoreLst) {
1326 deleteTree();
1327 UserIgnoreList = UserIgnoreLst;
1328 if (!allSameType(Roots))
1329 return;
1330 buildTree_rec(Roots, 0, -1);
1331
1332 // Collect the values that we need to extract from the tree.
1333 for (TreeEntry &EIdx : VectorizableTree) {
1334 TreeEntry *Entry = &EIdx;
1335
1336 // No need to handle users of gathered values.
1337 if (Entry->NeedToGather)
1338 continue;
1339
1340 // For each lane:
1341 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1342 Value *Scalar = Entry->Scalars[Lane];
1343
1344 // Check if the scalar is externally used as an extra arg.
1345 auto ExtI = ExternallyUsedValues.find(Scalar);
1346 if (ExtI != ExternallyUsedValues.end()) {
1347 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
1348 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false);
1349 ExternalUses.emplace_back(Scalar, nullptr, Lane);
1350 continue;
1351 }
1352 for (User *U : Scalar->users()) {
1353 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false);
1354
1355 Instruction *UserInst = dyn_cast<Instruction>(U);
1356 if (!UserInst)
1357 continue;
1358
1359 // Skip in-tree scalars that become vectors
1360 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1361 Value *UseScalar = UseEntry->Scalars[0];
1362 // Some in-tree scalars will remain as scalar in vectorized
1363 // instructions. If that is the case, the one in Lane 0 will
1364 // be used.
1365 if (UseScalar != U ||
1366 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1367 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1368 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false);
1369 assert(!UseEntry->NeedToGather && "Bad state")(static_cast <bool> (!UseEntry->NeedToGather &&
"Bad state") ? void (0) : __assert_fail ("!UseEntry->NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1369, __extension__ __PRETTY_FUNCTION__));
1370 continue;
1371 }
1372 }
1373
1374 // Ignore users in the user ignore list.
1375 if (is_contained(UserIgnoreList, UserInst))
1376 continue;
1377
1378 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1379 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false);
1380 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1381 }
1382 }
1383 }
1384}
1385
1386void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1387 int UserTreeIdx) {
1388 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(static_cast <bool> ((allConstant(VL) || allSameType(VL
)) && "Invalid types!") ? void (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1388, __extension__ __PRETTY_FUNCTION__));
1389
1390 InstructionsState S = getSameOpcode(VL);
1391 if (Depth == RecursionMaxDepth) {
1392 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false);
1393 newTreeEntry(VL, false, UserTreeIdx);
1394 return;
1395 }
1396
1397 // Don't handle vectors.
1398 if (S.OpValue->getType()->isVectorTy()) {
1399 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false);
1400 newTreeEntry(VL, false, UserTreeIdx);
1401 return;
1402 }
1403
1404 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1405 if (SI->getValueOperand()->getType()->isVectorTy()) {
1406 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false);
1407 newTreeEntry(VL, false, UserTreeIdx);
1408 return;
1409 }
1410
1411 // If all of the operands are identical or constant we have a simple solution.
1412 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1413 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false);
1414 newTreeEntry(VL, false, UserTreeIdx);
1415 return;
1416 }
1417
1418 // We now know that this is a vector of instructions of the same type from
1419 // the same block.
1420
1421 // Don't vectorize ephemeral values.
1422 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1423 if (EphValues.count(VL[i])) {
1424 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1425 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false);
1426 newTreeEntry(VL, false, UserTreeIdx);
1427 return;
1428 }
1429 }
1430
1431 // Check if this is a duplicate of another entry.
1432 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1433 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1434 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*VL[i] << ".\n"; } } while (false);
1435 if (E->Scalars[i] != VL[i]) {
1436 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false);
1437 newTreeEntry(VL, false, UserTreeIdx);
1438 return;
1439 }
1440 }
1441 // Record the reuse of the tree node. FIXME, currently this is only used to
1442 // properly draw the graph rather than for the actual vectorization.
1443 E->UserTreeIndices.push_back(UserTreeIdx);
1444 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false);
1445 return;
1446 }
1447
1448 // Check that none of the instructions in the bundle are already in the tree.
1449 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1450 auto *I = dyn_cast<Instruction>(VL[i]);
1451 if (!I)
1452 continue;
1453 if (getTreeEntry(I)) {
1454 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1455 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false);
1456 newTreeEntry(VL, false, UserTreeIdx);
1457 return;
1458 }
1459 }
1460
1461 // If any of the scalars is marked as a value that needs to stay scalar, then
1462 // we need to gather the scalars.
1463 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1464 if (MustGather.count(VL[i])) {
1465 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false);
1466 newTreeEntry(VL, false, UserTreeIdx);
1467 return;
1468 }
1469 }
1470
1471 // Check that all of the users of the scalars that we want to vectorize are
1472 // schedulable.
1473 auto *VL0 = cast<Instruction>(S.OpValue);
1474 BasicBlock *BB = VL0->getParent();
1475
1476 if (!DT->isReachableFromEntry(BB)) {
1477 // Don't go into unreachable blocks. They may contain instructions with
1478 // dependency cycles which confuse the final scheduling.
1479 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false);
1480 newTreeEntry(VL, false, UserTreeIdx);
1481 return;
1482 }
1483
1484 // Check that every instruction appears once in this bundle.
1485 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1486 for (unsigned j = i + 1; j < e; ++j)
1487 if (VL[i] == VL[j]) {
1488 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false);
1489 newTreeEntry(VL, false, UserTreeIdx);
1490 return;
1491 }
1492
1493 auto &BSRef = BlocksSchedules[BB];
1494 if (!BSRef)
1495 BSRef = llvm::make_unique<BlockScheduling>(BB);
1496
1497 BlockScheduling &BS = *BSRef.get();
1498
1499 if (!BS.tryScheduleBundle(VL, this, S.OpValue)) {
1500 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false);
1501 assert((!BS.getScheduleData(VL0) ||(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1503, __extension__ __PRETTY_FUNCTION__))
1502 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1503, __extension__ __PRETTY_FUNCTION__))
1503 "tryScheduleBundle should cancelScheduling on failure")(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1503, __extension__ __PRETTY_FUNCTION__));
1504 newTreeEntry(VL, false, UserTreeIdx);
1505 return;
1506 }
1507 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false);
1508
1509 unsigned ShuffleOrOp = S.IsAltShuffle ?
1510 (unsigned) Instruction::ShuffleVector : S.Opcode;
1511 switch (ShuffleOrOp) {
1512 case Instruction::PHI: {
1513 PHINode *PH = dyn_cast<PHINode>(VL0);
1514
1515 // Check for terminator values (e.g. invoke).
1516 for (unsigned j = 0; j < VL.size(); ++j)
1517 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1518 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1519 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1520 if (Term) {
1521 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false);
1522 BS.cancelScheduling(VL, VL0);
1523 newTreeEntry(VL, false, UserTreeIdx);
1524 return;
1525 }
1526 }
1527
1528 newTreeEntry(VL, true, UserTreeIdx);
1529 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false);
1530
1531 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1532 ValueList Operands;
1533 // Prepare the operand vector.
1534 for (Value *j : VL)
1535 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1536 PH->getIncomingBlock(i)));
1537
1538 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1539 }
1540 return;
1541 }
1542 case Instruction::ExtractValue:
1543 case Instruction::ExtractElement: {
1544 bool Reuse = canReuseExtract(VL, VL0);
1545 if (Reuse) {
1546 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing extract sequence.\n"
; } } while (false);
1547 } else {
1548 BS.cancelScheduling(VL, VL0);
1549 }
1550 newTreeEntry(VL, Reuse, UserTreeIdx);
1551 return;
1552 }
1553 case Instruction::Load: {
1554 // Check that a vectorized load would load the same memory as a scalar
1555 // load. For example, we don't want to vectorize loads that are smaller
1556 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1557 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1558 // from such a struct, we read/write packed bits disagreeing with the
1559 // unvectorized version.
1560 Type *ScalarTy = VL0->getType();
1561
1562 if (DL->getTypeSizeInBits(ScalarTy) !=
1563 DL->getTypeAllocSizeInBits(ScalarTy)) {
1564 BS.cancelScheduling(VL, VL0);
1565 newTreeEntry(VL, false, UserTreeIdx);
1566 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false);
1567 return;
1568 }
1569
1570 // Make sure all loads in the bundle are simple - we can't vectorize
1571 // atomic or volatile loads.
1572 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1573 LoadInst *L = cast<LoadInst>(VL[i]);
1574 if (!L->isSimple()) {
1575 BS.cancelScheduling(VL, VL0);
1576 newTreeEntry(VL, false, UserTreeIdx);
1577 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false);
1578 return;
1579 }
1580 }
1581
1582 // Check if the loads are consecutive, reversed, or neither.
1583 // TODO: What we really want is to sort the loads, but for now, check
1584 // the two likely directions.
1585 bool Consecutive = true;
1586 bool ReverseConsecutive = true;
1587 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1588 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1589 Consecutive = false;
1590 break;
1591 } else {
1592 ReverseConsecutive = false;
1593 }
1594 }
1595
1596 if (Consecutive) {
1597 ++NumLoadsWantToKeepOrder;
1598 newTreeEntry(VL, true, UserTreeIdx);
1599 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false);
1600 return;
1601 }
1602
1603 // If none of the load pairs were consecutive when checked in order,
1604 // check the reverse order.
1605 if (ReverseConsecutive)
1606 for (unsigned i = VL.size() - 1; i > 0; --i)
1607 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1608 ReverseConsecutive = false;
1609 break;
1610 }
1611
1612 BS.cancelScheduling(VL, VL0);
1613 newTreeEntry(VL, false, UserTreeIdx);
1614
1615 if (ReverseConsecutive) {
1616 ++NumLoadsWantToChangeOrder;
1617 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering reversed loads.\n"
; } } while (false);
1618 } else {
1619 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false);
1620 }
1621 return;
1622 }
1623 case Instruction::ZExt:
1624 case Instruction::SExt:
1625 case Instruction::FPToUI:
1626 case Instruction::FPToSI:
1627 case Instruction::FPExt:
1628 case Instruction::PtrToInt:
1629 case Instruction::IntToPtr:
1630 case Instruction::SIToFP:
1631 case Instruction::UIToFP:
1632 case Instruction::Trunc:
1633 case Instruction::FPTrunc:
1634 case Instruction::BitCast: {
1635 Type *SrcTy = VL0->getOperand(0)->getType();
1636 for (unsigned i = 0; i < VL.size(); ++i) {
1637 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1638 if (Ty != SrcTy || !isValidElementType(Ty)) {
1639 BS.cancelScheduling(VL, VL0);
1640 newTreeEntry(VL, false, UserTreeIdx);
1641 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false);
1642 return;
1643 }
1644 }
1645 newTreeEntry(VL, true, UserTreeIdx);
1646 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false);
1647
1648 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1649 ValueList Operands;
1650 // Prepare the operand vector.
1651 for (Value *j : VL)
1652 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1653
1654 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1655 }
1656 return;
1657 }
1658 case Instruction::ICmp:
1659 case Instruction::FCmp: {
1660 // Check that all of the compares have the same predicate.
1661 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1662 Type *ComparedTy = VL0->getOperand(0)->getType();
1663 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1664 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1665 if (Cmp->getPredicate() != P0 ||
1666 Cmp->getOperand(0)->getType() != ComparedTy) {
1667 BS.cancelScheduling(VL, VL0);
1668 newTreeEntry(VL, false, UserTreeIdx);
1669 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false);
1670 return;
1671 }
1672 }
1673
1674 newTreeEntry(VL, true, UserTreeIdx);
1675 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false);
1676
1677 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1678 ValueList Operands;
1679 // Prepare the operand vector.
1680 for (Value *j : VL)
1681 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1682
1683 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1684 }
1685 return;
1686 }
1687 case Instruction::Select:
1688 case Instruction::Add:
1689 case Instruction::FAdd:
1690 case Instruction::Sub:
1691 case Instruction::FSub:
1692 case Instruction::Mul:
1693 case Instruction::FMul:
1694 case Instruction::UDiv:
1695 case Instruction::SDiv:
1696 case Instruction::FDiv:
1697 case Instruction::URem:
1698 case Instruction::SRem:
1699 case Instruction::FRem:
1700 case Instruction::Shl:
1701 case Instruction::LShr:
1702 case Instruction::AShr:
1703 case Instruction::And:
1704 case Instruction::Or:
1705 case Instruction::Xor:
1706 newTreeEntry(VL, true, UserTreeIdx);
1707 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false);
1708
1709 // Sort operands of the instructions so that each side is more likely to
1710 // have the same opcode.
1711 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1712 ValueList Left, Right;
1713 reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1714 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1715 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1716 return;
1717 }
1718
1719 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1720 ValueList Operands;
1721 // Prepare the operand vector.
1722 for (Value *j : VL)
1723 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1724
1725 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1726 }
1727 return;
1728
1729 case Instruction::GetElementPtr: {
1730 // We don't combine GEPs with complicated (nested) indexing.
1731 for (unsigned j = 0; j < VL.size(); ++j) {
1732 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1733 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false);
1734 BS.cancelScheduling(VL, VL0);
1735 newTreeEntry(VL, false, UserTreeIdx);
1736 return;
1737 }
1738 }
1739
1740 // We can't combine several GEPs into one vector if they operate on
1741 // different types.
1742 Type *Ty0 = VL0->getOperand(0)->getType();
1743 for (unsigned j = 0; j < VL.size(); ++j) {
1744 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1745 if (Ty0 != CurTy) {
1746 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false);
1747 BS.cancelScheduling(VL, VL0);
1748 newTreeEntry(VL, false, UserTreeIdx);
1749 return;
1750 }
1751 }
1752
1753 // We don't combine GEPs with non-constant indexes.
1754 for (unsigned j = 0; j < VL.size(); ++j) {
1755 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1756 if (!isa<ConstantInt>(Op)) {
1757 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1758 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false);
1759 BS.cancelScheduling(VL, VL0);
1760 newTreeEntry(VL, false, UserTreeIdx);
1761 return;
1762 }
1763 }
1764
1765 newTreeEntry(VL, true, UserTreeIdx);
1766 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false);
1767 for (unsigned i = 0, e = 2; i < e; ++i) {
1768 ValueList Operands;
1769 // Prepare the operand vector.
1770 for (Value *j : VL)
1771 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1772
1773 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1774 }
1775 return;
1776 }
1777 case Instruction::Store: {
1778 // Check if the stores are consecutive or of we need to swizzle them.
1779 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1780 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1781 BS.cancelScheduling(VL, VL0);
1782 newTreeEntry(VL, false, UserTreeIdx);
1783 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false);
1784 return;
1785 }
1786
1787 newTreeEntry(VL, true, UserTreeIdx);
1788 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false);
1789
1790 ValueList Operands;
1791 for (Value *j : VL)
1792 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1793
1794 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1795 return;
1796 }
1797 case Instruction::Call: {
1798 // Check if the calls are all to the same vectorizable intrinsic.
1799 CallInst *CI = cast<CallInst>(VL0);
1800 // Check if this is an Intrinsic call or something that can be
1801 // represented by an intrinsic call
1802 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1803 if (!isTriviallyVectorizable(ID)) {
1804 BS.cancelScheduling(VL, VL0);
1805 newTreeEntry(VL, false, UserTreeIdx);
1806 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false);
1807 return;
1808 }
1809 Function *Int = CI->getCalledFunction();
1810 Value *A1I = nullptr;
1811 if (hasVectorInstrinsicScalarOpd(ID, 1))
1812 A1I = CI->getArgOperand(1);
1813 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1814 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1815 if (!CI2 || CI2->getCalledFunction() != Int ||
1816 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1817 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1818 BS.cancelScheduling(VL, VL0);
1819 newTreeEntry(VL, false, UserTreeIdx);
1820 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1821 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
);
1822 return;
1823 }
1824 // ctlz,cttz and powi are special intrinsics whose second argument
1825 // should be same in order for them to be vectorized.
1826 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1827 Value *A1J = CI2->getArgOperand(1);
1828 if (A1I != A1J) {
1829 BS.cancelScheduling(VL, VL0);
1830 newTreeEntry(VL, false, UserTreeIdx);
1831 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1832 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1833 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false);
1834 return;
1835 }
1836 }
1837 // Verify that the bundle operands are identical between the two calls.
1838 if (CI->hasOperandBundles() &&
1839 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1840 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1841 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1842 BS.cancelScheduling(VL, VL0);
1843 newTreeEntry(VL, false, UserTreeIdx);
1844 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1845 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false);
1846 return;
1847 }
1848 }
1849
1850 newTreeEntry(VL, true, UserTreeIdx);
1851 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1852 ValueList Operands;
1853 // Prepare the operand vector.
1854 for (Value *j : VL) {
1855 CallInst *CI2 = dyn_cast<CallInst>(j);
1856 Operands.push_back(CI2->getArgOperand(i));
1857 }
1858 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1859 }
1860 return;
1861 }
1862 case Instruction::ShuffleVector:
1863 // If this is not an alternate sequence of opcode like add-sub
1864 // then do not vectorize this instruction.
1865 if (!S.IsAltShuffle) {
1866 BS.cancelScheduling(VL, VL0);
1867 newTreeEntry(VL, false, UserTreeIdx);
1868 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false);
1869 return;
1870 }
1871 newTreeEntry(VL, true, UserTreeIdx);
1872 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false);
1873
1874 // Reorder operands if reordering would enable vectorization.
1875 if (isa<BinaryOperator>(VL0)) {
1876 ValueList Left, Right;
1877 reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1878 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1879 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1880 return;
1881 }
1882
1883 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1884 ValueList Operands;
1885 // Prepare the operand vector.
1886 for (Value *j : VL)
1887 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1888
1889 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1890 }
1891 return;
1892
1893 default:
1894 BS.cancelScheduling(VL, VL0);
1895 newTreeEntry(VL, false, UserTreeIdx);
1896 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false);
1897 return;
1898 }
1899}
1900
1901unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1902 unsigned N;
1903 Type *EltTy;
1904 auto *ST = dyn_cast<StructType>(T);
1905 if (ST) {
1906 N = ST->getNumElements();
1907 EltTy = *ST->element_begin();
1908 } else {
1909 N = cast<ArrayType>(T)->getNumElements();
1910 EltTy = cast<ArrayType>(T)->getElementType();
1911 }
1912 if (!isValidElementType(EltTy))
1913 return 0;
1914 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1915 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1916 return 0;
1917 if (ST) {
1918 // Check that struct is homogeneous.
1919 for (const auto *Ty : ST->elements())
1920 if (Ty != EltTy)
1921 return 0;
1922 }
1923 return N;
1924}
1925
1926bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const {
1927 Instruction *E0 = cast<Instruction>(OpValue);
1928 assert(E0->getOpcode() == Instruction::ExtractElement ||(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1929, __extension__ __PRETTY_FUNCTION__))
1929 E0->getOpcode() == Instruction::ExtractValue)(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1929, __extension__ __PRETTY_FUNCTION__));
1930 assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode")(static_cast <bool> (E0->getOpcode() == getSameOpcode
(VL).Opcode && "Invalid opcode") ? void (0) : __assert_fail
("E0->getOpcode() == getSameOpcode(VL).Opcode && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1930, __extension__ __PRETTY_FUNCTION__));
1931 // Check if all of the extracts come from the same vector and from the
1932 // correct offset.
1933 Value *Vec = E0->getOperand(0);
1934
1935 // We have to extract from a vector/aggregate with the same number of elements.
1936 unsigned NElts;
1937 if (E0->getOpcode() == Instruction::ExtractValue) {
1938 const DataLayout &DL = E0->getModule()->getDataLayout();
1939 NElts = canMapToVector(Vec->getType(), DL);
1940 if (!NElts)
1941 return false;
1942 // Check if load can be rewritten as load of vector.
1943 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1944 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1945 return false;
1946 } else {
1947 NElts = Vec->getType()->getVectorNumElements();
1948 }
1949
1950 if (NElts != VL.size())
1951 return false;
1952
1953 // Check that all of the indices extract from the correct offset.
1954 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
1955 Instruction *Inst = cast<Instruction>(VL[I]);
1956 if (!matchExtractIndex(Inst, I, Inst->getOpcode()))
1957 return false;
1958 if (Inst->getOperand(0) != Vec)
1959 return false;
1960 }
1961
1962 return true;
1963}
1964
1965bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
1966 return I->hasOneUse() ||
1967 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
1968 return ScalarToTreeEntry.count(U) > 0;
1969 });
1970}
1971
1972int BoUpSLP::getEntryCost(TreeEntry *E) {
1973 ArrayRef<Value*> VL = E->Scalars;
1974
1975 Type *ScalarTy = VL[0]->getType();
1976 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1977 ScalarTy = SI->getValueOperand()->getType();
1978 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
1979 ScalarTy = CI->getOperand(0)->getType();
1980 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1981
1982 // If we have computed a smaller type for the expression, update VecTy so
1983 // that the costs will be accurate.
1984 if (MinBWs.count(VL[0]))
1985 VecTy = VectorType::get(
1986 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1987
1988 if (E->NeedToGather) {
1989 if (allConstant(VL))
1990 return 0;
1991 if (isSplat(VL)) {
1992 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1993 }
1994 if (getSameOpcode(VL).Opcode == Instruction::ExtractElement) {
1995 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
1996 if (ShuffleKind.hasValue()) {
1997 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
1998 for (auto *V : VL) {
1999 // If all users of instruction are going to be vectorized and this
2000 // instruction itself is not going to be vectorized, consider this
2001 // instruction as dead and remove its cost from the final cost of the
2002 // vectorized tree.
2003 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2004 !ScalarToTreeEntry.count(V)) {
2005 auto *IO = cast<ConstantInt>(
2006 cast<ExtractElementInst>(V)->getIndexOperand());
2007 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2008 IO->getZExtValue());
2009 }
2010 }
2011 return Cost;
2012 }
2013 }
2014 return getGatherCost(E->Scalars);
2015 }
2016 InstructionsState S = getSameOpcode(VL);
2017 assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")(static_cast <bool> (S.Opcode && allSameType(VL
) && allSameBlock(VL) && "Invalid VL") ? void
(0) : __assert_fail ("S.Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2017, __extension__ __PRETTY_FUNCTION__));
2018 Instruction *VL0 = cast<Instruction>(S.OpValue);
2019 unsigned ShuffleOrOp = S.IsAltShuffle ?
2020 (unsigned) Instruction::ShuffleVector : S.Opcode;
2021 switch (ShuffleOrOp) {
2022 case Instruction::PHI:
2023 return 0;
2024
2025 case Instruction::ExtractValue:
2026 case Instruction::ExtractElement:
2027 if (canReuseExtract(VL, S.OpValue)) {
2028 int DeadCost = 0;
2029 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2030 Instruction *E = cast<Instruction>(VL[i]);
2031 // If all users are going to be vectorized, instruction can be
2032 // considered as dead.
2033 // The same, if have only one user, it will be vectorized for sure.
2034 if (areAllUsersVectorized(E))
2035 // Take credit for instruction that will become dead.
2036 DeadCost +=
2037 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2038 }
2039 return -DeadCost;
2040 }
2041 return getGatherCost(VecTy);
2042
2043 case Instruction::ZExt:
2044 case Instruction::SExt:
2045 case Instruction::FPToUI:
2046 case Instruction::FPToSI:
2047 case Instruction::FPExt:
2048 case Instruction::PtrToInt:
2049 case Instruction::IntToPtr:
2050 case Instruction::SIToFP:
2051 case Instruction::UIToFP:
2052 case Instruction::Trunc:
2053 case Instruction::FPTrunc:
2054 case Instruction::BitCast: {
2055 Type *SrcTy = VL0->getOperand(0)->getType();
2056
2057 // Calculate the cost of this instruction.
2058 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2059 VL0->getType(), SrcTy, VL0);
2060
2061 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2062 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2063 return VecCost - ScalarCost;
2064 }
2065 case Instruction::FCmp:
2066 case Instruction::ICmp:
2067 case Instruction::Select: {
2068 // Calculate the cost of this instruction.
2069 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2070 int ScalarCost = VecTy->getNumElements() *
2071 TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2072 int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2073 return VecCost - ScalarCost;
2074 }
2075 case Instruction::Add:
2076 case Instruction::FAdd:
2077 case Instruction::Sub:
2078 case Instruction::FSub:
2079 case Instruction::Mul:
2080 case Instruction::FMul:
2081 case Instruction::UDiv:
2082 case Instruction::SDiv:
2083 case Instruction::FDiv:
2084 case Instruction::URem:
2085 case Instruction::SRem:
2086 case Instruction::FRem:
2087 case Instruction::Shl:
2088 case Instruction::LShr:
2089 case Instruction::AShr:
2090 case Instruction::And:
2091 case Instruction::Or:
2092 case Instruction::Xor: {
2093 // Certain instructions can be cheaper to vectorize if they have a
2094 // constant second vector operand.
2095 TargetTransformInfo::OperandValueKind Op1VK =
2096 TargetTransformInfo::OK_AnyValue;
2097 TargetTransformInfo::OperandValueKind Op2VK =
2098 TargetTransformInfo::OK_UniformConstantValue;
2099 TargetTransformInfo::OperandValueProperties Op1VP =
2100 TargetTransformInfo::OP_None;
2101 TargetTransformInfo::OperandValueProperties Op2VP =
2102 TargetTransformInfo::OP_None;
2103
2104 // If all operands are exactly the same ConstantInt then set the
2105 // operand kind to OK_UniformConstantValue.
2106 // If instead not all operands are constants, then set the operand kind
2107 // to OK_AnyValue. If all operands are constants but not the same,
2108 // then set the operand kind to OK_NonUniformConstantValue.
2109 ConstantInt *CInt = nullptr;
2110 for (unsigned i = 0; i < VL.size(); ++i) {
2111 const Instruction *I = cast<Instruction>(VL[i]);
2112 if (!isa<ConstantInt>(I->getOperand(1))) {
2113 Op2VK = TargetTransformInfo::OK_AnyValue;
2114 break;
2115 }
2116 if (i == 0) {
2117 CInt = cast<ConstantInt>(I->getOperand(1));
2118 continue;
2119 }
2120 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
2121 CInt != cast<ConstantInt>(I->getOperand(1)))
2122 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2123 }
2124 // FIXME: Currently cost of model modification for division by power of
2125 // 2 is handled for X86 and AArch64. Add support for other targets.
2126 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2127 CInt->getValue().isPowerOf2())
2128 Op2VP = TargetTransformInfo::OP_PowerOf2;
2129
2130 SmallVector<const Value *, 4> Operands(VL0->operand_values());
2131 int ScalarCost =
2132 VecTy->getNumElements() *
2133 TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2134 Op2VP, Operands);
2135 int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2136 Op1VP, Op2VP, Operands);
2137 return VecCost - ScalarCost;
2138 }
2139 case Instruction::GetElementPtr: {
2140 TargetTransformInfo::OperandValueKind Op1VK =
2141 TargetTransformInfo::OK_AnyValue;
2142 TargetTransformInfo::OperandValueKind Op2VK =
2143 TargetTransformInfo::OK_UniformConstantValue;
2144
2145 int ScalarCost =
2146 VecTy->getNumElements() *
2147 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2148 int VecCost =
2149 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2150
2151 return VecCost - ScalarCost;
2152 }
2153 case Instruction::Load: {
2154 // Cost of wide load - cost of scalar loads.
2155 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2156 int ScalarLdCost = VecTy->getNumElements() *
2157 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2158 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2159 VecTy, alignment, 0, VL0);
2160 return VecLdCost - ScalarLdCost;
2161 }
2162 case Instruction::Store: {
2163 // We know that we can merge the stores. Calculate the cost.
2164 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2165 int ScalarStCost = VecTy->getNumElements() *
2166 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2167 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2168 VecTy, alignment, 0, VL0);
2169 return VecStCost - ScalarStCost;
2170 }
2171 case Instruction::Call: {
2172 CallInst *CI = cast<CallInst>(VL0);
2173 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2174
2175 // Calculate the cost of the scalar and vector calls.
2176 SmallVector<Type*, 4> ScalarTys;
2177 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2178 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2179
2180 FastMathFlags FMF;
2181 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2182 FMF = FPMO->getFastMathFlags();
2183
2184 int ScalarCallCost = VecTy->getNumElements() *
2185 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2186
2187 SmallVector<Value *, 4> Args(CI->arg_operands());
2188 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2189 VecTy->getNumElements());
2190
2191 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2192 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2193 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false);
2194
2195 return VecCallCost - ScalarCallCost;
2196 }
2197 case Instruction::ShuffleVector: {
2198 TargetTransformInfo::OperandValueKind Op1VK =
2199 TargetTransformInfo::OK_AnyValue;
2200 TargetTransformInfo::OperandValueKind Op2VK =
2201 TargetTransformInfo::OK_AnyValue;
2202 int ScalarCost = 0;
2203 int VecCost = 0;
2204 for (Value *i : VL) {
2205 Instruction *I = cast<Instruction>(i);
2206 if (!I)
2207 break;
2208 ScalarCost +=
2209 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2210 }
2211 // VecCost is equal to sum of the cost of creating 2 vectors
2212 // and the cost of creating shuffle.
2213 Instruction *I0 = cast<Instruction>(VL[0]);
2214 VecCost =
2215 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2216 Instruction *I1 = cast<Instruction>(VL[1]);
2217 VecCost +=
2218 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2219 VecCost +=
2220 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2221 return VecCost - ScalarCost;
2222 }
2223 default:
2224 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2224);
2225 }
2226}
2227
2228bool BoUpSLP::isFullyVectorizableTinyTree() {
2229 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
2230 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false);
2231
2232 // We only handle trees of heights 1 and 2.
2233 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2234 return true;
2235
2236 if (VectorizableTree.size() != 2)
2237 return false;
2238
2239 // Handle splat and all-constants stores.
2240 if (!VectorizableTree[0].NeedToGather &&
2241 (allConstant(VectorizableTree[1].Scalars) ||
2242 isSplat(VectorizableTree[1].Scalars)))
2243 return true;
2244
2245 // Gathering cost would be too much for tiny trees.
2246 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2247 return false;
2248
2249 return true;
2250}
2251
2252bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2253 // We can vectorize the tree if its size is greater than or equal to the
2254 // minimum size specified by the MinTreeSize command line option.
2255 if (VectorizableTree.size() >= MinTreeSize)
2256 return false;
2257
2258 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2259 // can vectorize it if we can prove it fully vectorizable.
2260 if (isFullyVectorizableTinyTree())
2261 return false;
2262
2263 assert(VectorizableTree.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2265, __extension__ __PRETTY_FUNCTION__))
2264 ? ExternalUses.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2265, __extension__ __PRETTY_FUNCTION__))
2265 : true && "We shouldn't have any external users")(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2265, __extension__ __PRETTY_FUNCTION__));
2266
2267 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2268 // vectorizable.
2269 return true;
2270}
2271
2272int BoUpSLP::getSpillCost() {
2273 // Walk from the bottom of the tree to the top, tracking which values are
2274 // live. When we see a call instruction that is not part of our tree,
2275 // query TTI to see if there is a cost to keeping values live over it
2276 // (for example, if spills and fills are required).
2277 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2278 int Cost = 0;
2279
2280 SmallPtrSet<Instruction*, 4> LiveValues;
2281 Instruction *PrevInst = nullptr;
2282
2283 for (const auto &N : VectorizableTree) {
2284 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2285 if (!Inst)
2286 continue;
2287
2288 if (!PrevInst) {
2289 PrevInst = Inst;
2290 continue;
2291 }
2292
2293 // Update LiveValues.
2294 LiveValues.erase(PrevInst);
2295 for (auto &J : PrevInst->operands()) {
2296 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2297 LiveValues.insert(cast<Instruction>(&*J));
2298 }
2299
2300 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2301 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2302 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2303 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2304 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2305 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2306 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false);
2307
2308 // Now find the sequence of instructions between PrevInst and Inst.
2309 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2310 PrevInstIt =
2311 PrevInst->getIterator().getReverse();
2312 while (InstIt != PrevInstIt) {
2313 if (PrevInstIt == PrevInst->getParent()->rend()) {
2314 PrevInstIt = Inst->getParent()->rbegin();
2315 continue;
2316 }
2317
2318 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2319 SmallVector<Type*, 4> V;
2320 for (auto *II : LiveValues)
2321 V.push_back(VectorType::get(II->getType(), BundleWidth));
2322 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2323 }
2324
2325 ++PrevInstIt;
2326 }
2327
2328 PrevInst = Inst;
2329 }
2330
2331 return Cost;
2332}
2333
2334int BoUpSLP::getTreeCost() {
2335 int Cost = 0;
2336 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
2337 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false);
2338
2339 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2340
2341 for (TreeEntry &TE : VectorizableTree) {
2342 int C = getEntryCost(&TE);
2343 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
2344 << *TE.Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false);
2345 Cost += C;
2346 }
2347
2348 SmallSet<Value *, 16> ExtractCostCalculated;
2349 int ExtractCost = 0;
2350 for (ExternalUser &EU : ExternalUses) {
2351 // We only add extract cost once for the same scalar.
2352 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2353 continue;
2354
2355 // Uses by ephemeral values are free (because the ephemeral value will be
2356 // removed prior to code generation, and so the extraction will be
2357 // removed as well).
2358 if (EphValues.count(EU.User))
2359 continue;
2360
2361 // If we plan to rewrite the tree in a smaller type, we will need to sign
2362 // extend the extracted value back to the original type. Here, we account
2363 // for the extract and the added cost of the sign extend if needed.
2364 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2365 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2366 if (MinBWs.count(ScalarRoot)) {
2367 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2368 auto Extend =
2369 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2370 VecTy = VectorType::get(MinTy, BundleWidth);
2371 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2372 VecTy, EU.Lane);
2373 } else {
2374 ExtractCost +=
2375 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2376 }
2377 }
2378
2379 int SpillCost = getSpillCost();
2380 Cost += SpillCost + ExtractCost;
2381
2382 std::string Str;
2383 {
2384 raw_string_ostream OS(Str);
2385 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2386 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2387 << "SLP: Total Cost = " << Cost << ".\n";
2388 }
2389 DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false);
2390
2391 if (ViewSLPTree)
2392 ViewGraph(this, "SLP" + F->getName(), false, Str);
2393
2394 return Cost;
2395}
2396
2397int BoUpSLP::getGatherCost(Type *Ty) {
2398 int Cost = 0;
2399 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2400 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2401 return Cost;
2402}
2403
2404int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2405 // Find the type of the operands in VL.
2406 Type *ScalarTy = VL[0]->getType();
2407 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2408 ScalarTy = SI->getValueOperand()->getType();
2409 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2410 // Find the cost of inserting/extracting values from the vector.
2411 return getGatherCost(VecTy);
2412}
2413
2414// Reorder commutative operations in alternate shuffle if the resulting vectors
2415// are consecutive loads. This would allow us to vectorize the tree.
2416// If we have something like-
2417// load a[0] - load b[0]
2418// load b[1] + load a[1]
2419// load a[2] - load b[2]
2420// load a[3] + load b[3]
2421// Reordering the second load b[1] load a[1] would allow us to vectorize this
2422// code.
2423void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2424 SmallVectorImpl<Value *> &Left,
2425 SmallVectorImpl<Value *> &Right) {
2426 // Push left and right operands of binary operation into Left and Right
2427 unsigned AltOpcode = getAltOpcode(Opcode);
2428 (void)AltOpcode;
2429 for (Value *V : VL) {
2430 auto *I = cast<Instruction>(V);
2431 assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&(static_cast <bool> (sameOpcodeOrAlt(Opcode, AltOpcode,
I->getOpcode()) && "Incorrect instruction in vector"
) ? void (0) : __assert_fail ("sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2432, __extension__ __PRETTY_FUNCTION__))
2432 "Incorrect instruction in vector")(static_cast <bool> (sameOpcodeOrAlt(Opcode, AltOpcode,
I->getOpcode()) && "Incorrect instruction in vector"
) ? void (0) : __assert_fail ("sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2432, __extension__ __PRETTY_FUNCTION__));
2433 Left.push_back(I->getOperand(0));
2434 Right.push_back(I->getOperand(1));
2435 }
2436
2437 // Reorder if we have a commutative operation and consecutive access
2438 // are on either side of the alternate instructions.
2439 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2440 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2441 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2442 Instruction *VL1 = cast<Instruction>(VL[j]);
2443 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2444 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2445 std::swap(Left[j], Right[j]);
2446 continue;
2447 } else if (VL2->isCommutative() &&
2448 isConsecutiveAccess(L, L1, *DL, *SE)) {
2449 std::swap(Left[j + 1], Right[j + 1]);
2450 continue;
2451 }
2452 // else unchanged
2453 }
2454 }
2455 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2456 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2457 Instruction *VL1 = cast<Instruction>(VL[j]);
2458 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2459 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2460 std::swap(Left[j], Right[j]);
2461 continue;
2462 } else if (VL2->isCommutative() &&
2463 isConsecutiveAccess(L, L1, *DL, *SE)) {
2464 std::swap(Left[j + 1], Right[j + 1]);
2465 continue;
2466 }
2467 // else unchanged
2468 }
2469 }
2470 }
2471}
2472
2473// Return true if I should be commuted before adding it's left and right
2474// operands to the arrays Left and Right.
2475//
2476// The vectorizer is trying to either have all elements one side being
2477// instruction with the same opcode to enable further vectorization, or having
2478// a splat to lower the vectorizing cost.
2479static bool shouldReorderOperands(
2480 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2481 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2482 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2483 VLeft = I.getOperand(0);
2484 VRight = I.getOperand(1);
2485 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2486 if (SplatRight) {
2487 if (VRight == Right[i - 1])
2488 // Preserve SplatRight
2489 return false;
2490 if (VLeft == Right[i - 1]) {
2491 // Commuting would preserve SplatRight, but we don't want to break
2492 // SplatLeft either, i.e. preserve the original order if possible.
2493 // (FIXME: why do we care?)
2494 if (SplatLeft && VLeft == Left[i - 1])
2495 return false;
2496 return true;
2497 }
2498 }
2499 // Symmetrically handle Right side.
2500 if (SplatLeft) {
2501 if (VLeft == Left[i - 1])
2502 // Preserve SplatLeft
2503 return false;
2504 if (VRight == Left[i - 1])
2505 return true;
2506 }
2507
2508 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2509 Instruction *IRight = dyn_cast<Instruction>(VRight);
2510
2511 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2512 // it and not the right, in this case we want to commute.
2513 if (AllSameOpcodeRight) {
2514 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2515 if (IRight && RightPrevOpcode == IRight->getOpcode())
2516 // Do not commute, a match on the right preserves AllSameOpcodeRight
2517 return false;
2518 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2519 // We have a match and may want to commute, but first check if there is
2520 // not also a match on the existing operands on the Left to preserve
2521 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2522 // (FIXME: why do we care?)
2523 if (AllSameOpcodeLeft && ILeft &&
2524 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2525 return false;
2526 return true;
2527 }
2528 }
2529 // Symmetrically handle Left side.
2530 if (AllSameOpcodeLeft) {
2531 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2532 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2533 return false;
2534 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2535 return true;
2536 }
2537 return false;
2538}
2539
2540void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2541 ArrayRef<Value *> VL,
2542 SmallVectorImpl<Value *> &Left,
2543 SmallVectorImpl<Value *> &Right) {
2544 if (!VL.empty()) {
2545 // Peel the first iteration out of the loop since there's nothing
2546 // interesting to do anyway and it simplifies the checks in the loop.
2547 auto *I = cast<Instruction>(VL[0]);
2548 Value *VLeft = I->getOperand(0);
2549 Value *VRight = I->getOperand(1);
2550 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2551 // Favor having instruction to the right. FIXME: why?
2552 std::swap(VLeft, VRight);
2553 Left.push_back(VLeft);
2554 Right.push_back(VRight);
2555 }
2556
2557 // Keep track if we have instructions with all the same opcode on one side.
2558 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2559 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2560 // Keep track if we have one side with all the same value (broadcast).
2561 bool SplatLeft = true;
2562 bool SplatRight = true;
2563
2564 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2565 Instruction *I = cast<Instruction>(VL[i]);
2566 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2568, __extension__ __PRETTY_FUNCTION__))
2567 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2568, __extension__ __PRETTY_FUNCTION__))
2568 "Can only process commutative instruction")(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2568, __extension__ __PRETTY_FUNCTION__));
2569 // Commute to favor either a splat or maximizing having the same opcodes on
2570 // one side.
2571 Value *VLeft;
2572 Value *VRight;
2573 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2574 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2575 VRight)) {
2576 Left.push_back(VRight);
2577 Right.push_back(VLeft);
2578 } else {
2579 Left.push_back(VLeft);
2580 Right.push_back(VRight);
2581 }
2582 // Update Splat* and AllSameOpcode* after the insertion.
2583 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2584 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2585 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2586 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2587 cast<Instruction>(Left[i])->getOpcode());
2588 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2589 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2590 cast<Instruction>(Right[i])->getOpcode());
2591 }
2592
2593 // If one operand end up being broadcast, return this operand order.
2594 if (SplatRight || SplatLeft)
2595 return;
2596
2597 // Finally check if we can get longer vectorizable chain by reordering
2598 // without breaking the good operand order detected above.
2599 // E.g. If we have something like-
2600 // load a[0] load b[0]
2601 // load b[1] load a[1]
2602 // load a[2] load b[2]
2603 // load a[3] load b[3]
2604 // Reordering the second load b[1] load a[1] would allow us to vectorize
2605 // this code and we still retain AllSameOpcode property.
2606 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2607 // such as-
2608 // add a[0],c[0] load b[0]
2609 // add a[1],c[2] load b[1]
2610 // b[2] load b[2]
2611 // add a[3],c[3] load b[3]
2612 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2613 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2614 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2615 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2616 std::swap(Left[j + 1], Right[j + 1]);
2617 continue;
2618 }
2619 }
2620 }
2621 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2622 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2623 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2624 std::swap(Left[j + 1], Right[j + 1]);
2625 continue;
2626 }
2627 }
2628 }
2629 // else unchanged
2630 }
2631}
2632
2633void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2634 // Get the basic block this bundle is in. All instructions in the bundle
2635 // should be in this block.
2636 auto *Front = cast<Instruction>(OpValue);
2637 auto *BB = Front->getParent();
2638 const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2639 const unsigned AltOpcode = getAltOpcode(Opcode);
2640 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
2641 return !sameOpcodeOrAlt(Opcode, AltOpcode,(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
2642 cast<Instruction>(V)->getOpcode()) ||(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
2643 cast<Instruction>(V)->getParent() == BB;(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
2644 }))(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__));
2645
2646 // The last instruction in the bundle in program order.
2647 Instruction *LastInst = nullptr;
2648
2649 // Find the last instruction. The common case should be that BB has been
2650 // scheduled, and the last instruction is VL.back(). So we start with
2651 // VL.back() and iterate over schedule data until we reach the end of the
2652 // bundle. The end of the bundle is marked by null ScheduleData.
2653 if (BlocksSchedules.count(BB)) {
2654 auto *Bundle =
2655 BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2656 if (Bundle && Bundle->isPartOfBundle())
2657 for (; Bundle; Bundle = Bundle->NextInBundle)
2658 if (Bundle->OpValue == Bundle->Inst)
2659 LastInst = Bundle->Inst;
2660 }
2661
2662 // LastInst can still be null at this point if there's either not an entry
2663 // for BB in BlocksSchedules or there's no ScheduleData available for
2664 // VL.back(). This can be the case if buildTree_rec aborts for various
2665 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2666 // size is reached, etc.). ScheduleData is initialized in the scheduling
2667 // "dry-run".
2668 //
2669 // If this happens, we can still find the last instruction by brute force. We
2670 // iterate forwards from Front (inclusive) until we either see all
2671 // instructions in the bundle or reach the end of the block. If Front is the
2672 // last instruction in program order, LastInst will be set to Front, and we
2673 // will visit all the remaining instructions in the block.
2674 //
2675 // One of the reasons we exit early from buildTree_rec is to place an upper
2676 // bound on compile-time. Thus, taking an additional compile-time hit here is
2677 // not ideal. However, this should be exceedingly rare since it requires that
2678 // we both exit early from buildTree_rec and that the bundle be out-of-order
2679 // (causing us to iterate all the way to the end of the block).
2680 if (!LastInst) {
2681 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2682 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2683 if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2684 LastInst = &I;
2685 if (Bundle.empty())
2686 break;
2687 }
2688 }
2689
2690 // Set the insertion point after the last instruction in the bundle. Set the
2691 // debug location to Front.
2692 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2693 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2694}
2695
2696Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2697 Value *Vec = UndefValue::get(Ty);
2698 // Generate the 'InsertElement' instruction.
2699 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2700 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2701 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2702 GatherSeq.insert(Insrt);
2703 CSEBlocks.insert(Insrt->getParent());
2704
2705 // Add to our 'need-to-extract' list.
2706 if (TreeEntry *E = getTreeEntry(VL[i])) {
2707 // Find which lane we need to extract.
2708 int FoundLane = -1;
2709 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2710 // Is this the lane of the scalar that we are looking for ?
2711 if (E->Scalars[Lane] == VL[i]) {
2712 FoundLane = Lane;
2713 break;
2714 }
2715 }
2716 assert(FoundLane >= 0 && "Could not find the correct lane")(static_cast <bool> (FoundLane >= 0 && "Could not find the correct lane"
) ? void (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2716, __extension__ __PRETTY_FUNCTION__));
2717 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2718 }
2719 }
2720 }
2721
2722 return Vec;
2723}
2724
2725Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const {
2726 if (const TreeEntry *En = getTreeEntry(OpValue)) {
2727 if (En->isSame(VL) && En->VectorizedValue)
2728 return En->VectorizedValue;
2729 }
2730 return nullptr;
2731}
2732
2733Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2734 InstructionsState S = getSameOpcode(VL);
2735 if (S.Opcode) {
2736 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2737 if (E->isSame(VL))
2738 return vectorizeTree(E);
2739 }
2740 }
2741
2742 Type *ScalarTy = S.OpValue->getType();
2743 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2744 ScalarTy = SI->getValueOperand()->getType();
2745 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2746
2747 return Gather(VL, VecTy);
2748}
2749
2750Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2751 IRBuilder<>::InsertPointGuard Guard(Builder);
2752
2753 if (E->VectorizedValue) {
1
Assuming the condition is false
→
2
←
Taking false branch
→
2754 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false);
2755 return E->VectorizedValue;
2756 }
2757
2758 InstructionsState S = getSameOpcode(E->Scalars);
2759 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2760 Type *ScalarTy = VL0->getType();
2761 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3
←
Taking false branch
→
2762 ScalarTy = SI->getValueOperand()->getType();
2763 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2764
2765 if (E->NeedToGather) {
4
←
Assuming the condition is false
→
5
←
Taking false branch
→
2766 setInsertPointAfterBundle(E->Scalars, VL0);
2767 auto *V = Gather(E->Scalars, VecTy);
2768 E->VectorizedValue = V;
2769 return V;
2770 }
2771
2772 unsigned ShuffleOrOp = S.IsAltShuffle ?
6
←
'?' condition is false
→
2773 (unsigned) Instruction::ShuffleVector : S.Opcode;
2774 switch (ShuffleOrOp) {
7
←
Control jumps to 'case Call:' at line 3052
→
2775 case Instruction::PHI: {
2776 PHINode *PH = dyn_cast<PHINode>(VL0);
2777 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2778 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2779 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2780 E->VectorizedValue = NewPhi;
2781
2782 // PHINodes may have multiple entries from the same block. We want to
2783 // visit every block once.
2784 SmallSet<BasicBlock*, 4> VisitedBBs;
2785
2786 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2787 ValueList Operands;
2788 BasicBlock *IBB = PH->getIncomingBlock(i);
2789
2790 if (!VisitedBBs.insert(IBB).second) {
2791 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2792 continue;
2793 }
2794
2795 // Prepare the operand vector.
2796 for (Value *V : E->Scalars)
2797 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2798
2799 Builder.SetInsertPoint(IBB->getTerminator());
2800 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2801 Value *Vec = vectorizeTree(Operands);
2802 NewPhi->addIncoming(Vec, IBB);
2803 }
2804
2805 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2806, __extension__ __PRETTY_FUNCTION__))
2806 "Invalid number of incoming values")(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2806, __extension__ __PRETTY_FUNCTION__));
2807 return NewPhi;
2808 }
2809
2810 case Instruction::ExtractElement: {
2811 if (canReuseExtract(E->Scalars, VL0)) {
2812 Value *V = VL0->getOperand(0);
2813 E->VectorizedValue = V;
2814 return V;
2815 }
2816 setInsertPointAfterBundle(E->Scalars, VL0);
2817 auto *V = Gather(E->Scalars, VecTy);
2818 E->VectorizedValue = V;
2819 return V;
2820 }
2821 case Instruction::ExtractValue: {
2822 if (canReuseExtract(E->Scalars, VL0)) {
2823 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2824 Builder.SetInsertPoint(LI);
2825 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2826 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2827 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2828 E->VectorizedValue = V;
2829 return propagateMetadata(V, E->Scalars);
2830 }
2831 setInsertPointAfterBundle(E->Scalars, VL0);
2832 auto *V = Gather(E->Scalars, VecTy);
2833 E->VectorizedValue = V;
2834 return V;
2835 }
2836 case Instruction::ZExt:
2837 case Instruction::SExt:
2838 case Instruction::FPToUI:
2839 case Instruction::FPToSI:
2840 case Instruction::FPExt:
2841 case Instruction::PtrToInt:
2842 case Instruction::IntToPtr:
2843 case Instruction::SIToFP:
2844 case Instruction::UIToFP:
2845 case Instruction::Trunc:
2846 case Instruction::FPTrunc:
2847 case Instruction::BitCast: {
2848 ValueList INVL;
2849 for (Value *V : E->Scalars)
2850 INVL.push_back(cast<Instruction>(V)->getOperand(0));
2851
2852 setInsertPointAfterBundle(E->Scalars, VL0);
2853
2854 Value *InVec = vectorizeTree(INVL);
2855
2856 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2857 return V;
2858
2859 CastInst *CI = dyn_cast<CastInst>(VL0);
2860 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2861 E->VectorizedValue = V;
2862 ++NumVectorInstructions;
2863 return V;
2864 }
2865 case Instruction::FCmp:
2866 case Instruction::ICmp: {
2867 ValueList LHSV, RHSV;
2868 for (Value *V : E->Scalars) {
2869 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2870 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2871 }
2872
2873 setInsertPointAfterBundle(E->Scalars, VL0);
2874
2875 Value *L = vectorizeTree(LHSV);
2876 Value *R = vectorizeTree(RHSV);
2877
2878 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2879 return V;
2880
2881 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2882 Value *V;
2883 if (S.Opcode == Instruction::FCmp)
2884 V = Builder.CreateFCmp(P0, L, R);
2885 else
2886 V = Builder.CreateICmp(P0, L, R);
2887
2888 E->VectorizedValue = V;
2889 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2890 ++NumVectorInstructions;
2891 return V;
2892 }
2893 case Instruction::Select: {
2894 ValueList TrueVec, FalseVec, CondVec;
2895 for (Value *V : E->Scalars) {
2896 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2897 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2898 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2899 }
2900
2901 setInsertPointAfterBundle(E->Scalars, VL0);
2902
2903 Value *Cond = vectorizeTree(CondVec);
2904 Value *True = vectorizeTree(TrueVec);
2905 Value *False = vectorizeTree(FalseVec);
2906
2907 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2908 return V;
2909
2910 Value *V = Builder.CreateSelect(Cond, True, False);
2911 E->VectorizedValue = V;
2912 ++NumVectorInstructions;
2913 return V;
2914 }
2915 case Instruction::Add:
2916 case Instruction::FAdd:
2917 case Instruction::Sub:
2918 case Instruction::FSub:
2919 case Instruction::Mul:
2920 case Instruction::FMul:
2921 case Instruction::UDiv:
2922 case Instruction::SDiv:
2923 case Instruction::FDiv:
2924 case Instruction::URem:
2925 case Instruction::SRem:
2926 case Instruction::FRem:
2927 case Instruction::Shl:
2928 case Instruction::LShr:
2929 case Instruction::AShr:
2930 case Instruction::And:
2931 case Instruction::Or:
2932 case Instruction::Xor: {
2933 ValueList LHSVL, RHSVL;
2934 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2935 reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
2936 RHSVL);
2937 else
2938 for (Value *V : E->Scalars) {
2939 auto *I = cast<Instruction>(V);
2940 LHSVL.push_back(I->getOperand(0));
2941 RHSVL.push_back(I->getOperand(1));
2942 }
2943
2944 setInsertPointAfterBundle(E->Scalars, VL0);
2945
2946 Value *LHS = vectorizeTree(LHSVL);
2947 Value *RHS = vectorizeTree(RHSVL);
2948
2949 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2950 return V;
2951
2952 Value *V = Builder.CreateBinOp(
2953 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
2954 E->VectorizedValue = V;
2955 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2956 ++NumVectorInstructions;
2957
2958 if (Instruction *I = dyn_cast<Instruction>(V))
2959 return propagateMetadata(I, E->Scalars);
2960
2961 return V;
2962 }
2963 case Instruction::Load: {
2964 // Loads are inserted at the head of the tree because we don't want to
2965 // sink them all the way down past store instructions.
2966 setInsertPointAfterBundle(E->Scalars, VL0);
2967
2968 LoadInst *LI = cast<LoadInst>(VL0);
2969 Type *ScalarLoadTy = LI->getType();
2970 unsigned AS = LI->getPointerAddressSpace();
2971
2972 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2973 VecTy->getPointerTo(AS));
2974
2975 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2976 // ExternalUses list to make sure that an extract will be generated in the
2977 // future.
2978 Value *PO = LI->getPointerOperand();
2979 if (getTreeEntry(PO))
2980 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
2981
2982 unsigned Alignment = LI->getAlignment();
2983 LI = Builder.CreateLoad(VecPtr);
2984 if (!Alignment) {
2985 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2986 }
2987 LI->setAlignment(Alignment);
2988 E->VectorizedValue = LI;
2989 ++NumVectorInstructions;
2990 return propagateMetadata(LI, E->Scalars);
2991 }
2992 case Instruction::Store: {
2993 StoreInst *SI = cast<StoreInst>(VL0);
2994 unsigned Alignment = SI->getAlignment();
2995 unsigned AS = SI->getPointerAddressSpace();
2996
2997 ValueList ScalarStoreValues;
2998 for (Value *V : E->Scalars)
2999 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3000
3001 setInsertPointAfterBundle(E->Scalars, VL0);
3002
3003 Value *VecValue = vectorizeTree(ScalarStoreValues);
3004 Value *ScalarPtr = SI->getPointerOperand();
3005 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3006 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3007
3008 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3009 // ExternalUses to make sure that an extract will be generated in the
3010 // future.
3011 if (getTreeEntry(ScalarPtr))
3012 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3013
3014 if (!Alignment)
3015 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3016
3017 S->setAlignment(Alignment);
3018 E->VectorizedValue = S;
3019 ++NumVectorInstructions;
3020 return propagateMetadata(S, E->Scalars);
3021 }
3022 case Instruction::GetElementPtr: {
3023 setInsertPointAfterBundle(E->Scalars, VL0);
3024
3025 ValueList Op0VL;
3026 for (Value *V : E->Scalars)
3027 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3028
3029 Value *Op0 = vectorizeTree(Op0VL);
3030
3031 std::vector<Value *> OpVecs;
3032 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3033 ++j) {
3034 ValueList OpVL;
3035 for (Value *V : E->Scalars)
3036 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3037
3038 Value *OpVec = vectorizeTree(OpVL);
3039 OpVecs.push_back(OpVec);
3040 }
3041
3042 Value *V = Builder.CreateGEP(
3043 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3044 E->VectorizedValue = V;
3045 ++NumVectorInstructions;
3046
3047 if (Instruction *I = dyn_cast<Instruction>(V))
3048 return propagateMetadata(I, E->Scalars);
3049
3050 return V;
3051 }
3052 case Instruction::Call: {
3053 CallInst *CI = cast<CallInst>(VL0);
8
←
Calling 'cast'
→
32
←
Returning from 'cast'
→
33
←
'CI' initialized here
→
3054 setInsertPointAfterBundle(E->Scalars, VL0);
3055 Function *FI;
3056 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3057 Value *ScalarArg = nullptr;
3058 if (CI && (FI = CI->getCalledFunction())) {
34
←
Assuming 'CI' is null
→
3059 IID = FI->getIntrinsicID();
3060 }
3061 std::vector<Value *> OpVecs;
3062 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
35
←
Called C++ object pointer is null
3063 ValueList OpVL;
3064 // ctlz,cttz and powi are special intrinsics whose second argument is
3065 // a scalar. This argument should not be vectorized.
3066 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3067 CallInst *CEI = cast<CallInst>(VL0);
3068 ScalarArg = CEI->getArgOperand(j);
3069 OpVecs.push_back(CEI->getArgOperand(j));
3070 continue;
3071 }
3072 for (Value *V : E->Scalars) {
3073 CallInst *CEI = cast<CallInst>(V);
3074 OpVL.push_back(CEI->getArgOperand(j));
3075 }
3076
3077 Value *OpVec = vectorizeTree(OpVL);
3078 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false);
3079 OpVecs.push_back(OpVec);
3080 }
3081
3082 Module *M = F->getParent();
3083 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3084 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3085 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3086 SmallVector<OperandBundleDef, 1> OpBundles;
3087 CI->getOperandBundlesAsDefs(OpBundles);
3088 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3089
3090 // The scalar argument uses an in-tree scalar so we add the new vectorized
3091 // call to ExternalUses list to make sure that an extract will be
3092 // generated in the future.
3093 if (ScalarArg && getTreeEntry(ScalarArg))
3094 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3095
3096 E->VectorizedValue = V;
3097 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3098 ++NumVectorInstructions;
3099 return V;
3100 }
3101 case Instruction::ShuffleVector: {
3102 ValueList LHSVL, RHSVL;
3103 assert(Instruction::isBinaryOp(S.Opcode) &&(static_cast <bool> (Instruction::isBinaryOp(S.Opcode) &&
"Invalid Shuffle Vector Operand") ? void (0) : __assert_fail
("Instruction::isBinaryOp(S.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3104, __extension__ __PRETTY_FUNCTION__))
3104 "Invalid Shuffle Vector Operand")(static_cast <bool> (Instruction::isBinaryOp(S.Opcode) &&
"Invalid Shuffle Vector Operand") ? void (0) : __assert_fail
("Instruction::isBinaryOp(S.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3104, __extension__ __PRETTY_FUNCTION__));
3105 reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3106 setInsertPointAfterBundle(E->Scalars, VL0);
3107
3108 Value *LHS = vectorizeTree(LHSVL);
3109 Value *RHS = vectorizeTree(RHSVL);
3110
3111 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3112 return V;
3113
3114 // Create a vector of LHS op1 RHS
3115 Value *V0 = Builder.CreateBinOp(
3116 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3117
3118 unsigned AltOpcode = getAltOpcode(S.Opcode);
3119 // Create a vector of LHS op2 RHS
3120 Value *V1 = Builder.CreateBinOp(
3121 static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3122
3123 // Create shuffle to take alternate operations from the vector.
3124 // Also, gather up odd and even scalar ops to propagate IR flags to
3125 // each vector operation.
3126 ValueList OddScalars, EvenScalars;
3127 unsigned e = E->Scalars.size();
3128 SmallVector<Constant *, 8> Mask(e);
3129 for (unsigned i = 0; i < e; ++i) {
3130 if (isOdd(i)) {
3131 Mask[i] = Builder.getInt32(e + i);
3132 OddScalars.push_back(E->Scalars[i]);
3133 } else {
3134 Mask[i] = Builder.getInt32(i);
3135 EvenScalars.push_back(E->Scalars[i]);
3136 }
3137 }
3138
3139 Value *ShuffleMask = ConstantVector::get(Mask);
3140 propagateIRFlags(V0, EvenScalars);
3141 propagateIRFlags(V1, OddScalars);
3142
3143 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3144 E->VectorizedValue = V;
3145 ++NumVectorInstructions;
3146 if (Instruction *I = dyn_cast<Instruction>(V))
3147 return propagateMetadata(I, E->Scalars);
3148
3149 return V;
3150 }
3151 default:
3152 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3152);
3153 }
3154 return nullptr;
3155}
3156
3157Value *BoUpSLP::vectorizeTree() {
3158 ExtraValueToDebugLocsMap ExternallyUsedValues;
3159 return vectorizeTree(ExternallyUsedValues);
3160}
3161
3162Value *
3163BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3164 // All blocks must be scheduled before any instructions are inserted.
3165 for (auto &BSIter : BlocksSchedules) {
3166 scheduleBlock(BSIter.second.get());
3167 }
3168
3169 Builder.SetInsertPoint(&F->getEntryBlock().front());
3170 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3171
3172 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3173 // vectorized root. InstCombine will then rewrite the entire expression. We
3174 // sign extend the extracted values below.
3175 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3176 if (MinBWs.count(ScalarRoot)) {
3177 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3178 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3179 auto BundleWidth = VectorizableTree[0].Scalars.size();
3180 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3181 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3182 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3183 VectorizableTree[0].VectorizedValue = Trunc;
3184 }
3185
3186 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false);
3187
3188 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3189 // specified by ScalarType.
3190 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3191 if (!MinBWs.count(ScalarRoot))
3192 return Ex;
3193 if (MinBWs[ScalarRoot].second)
3194 return Builder.CreateSExt(Ex, ScalarType);
3195 return Builder.CreateZExt(Ex, ScalarType);
3196 };
3197
3198 // Extract all of the elements with the external uses.
3199 for (const auto &ExternalUse : ExternalUses) {
3200 Value *Scalar = ExternalUse.Scalar;
3201 llvm::User *User = ExternalUse.User;
3202
3203 // Skip users that we already RAUW. This happens when one instruction
3204 // has multiple uses of the same value.
3205 if (User && !is_contained(Scalar->users(), User))
3206 continue;
3207 TreeEntry *E = getTreeEntry(Scalar);
3208 assert(E && "Invalid scalar")(static_cast <bool> (E && "Invalid scalar") ? void
(0) : __assert_fail ("E && \"Invalid scalar\"", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3208, __extension__ __PRETTY_FUNCTION__));
3209 assert(!E->NeedToGather && "Extracting from a gather list")(static_cast <bool> (!E->NeedToGather && "Extracting from a gather list"
) ? void (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3209, __extension__ __PRETTY_FUNCTION__));
3210
3211 Value *Vec = E->VectorizedValue;
3212 assert(Vec && "Can't find vectorizable value")(static_cast <bool> (Vec && "Can't find vectorizable value"
) ? void (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3212, __extension__ __PRETTY_FUNCTION__));
3213
3214 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3215 // If User == nullptr, the Scalar is used as extra arg. Generate
3216 // ExtractElement instruction and update the record for this scalar in
3217 // ExternallyUsedValues.
3218 if (!User) {
3219 assert(ExternallyUsedValues.count(Scalar) &&(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3221, __extension__ __PRETTY_FUNCTION__))
3220 "Scalar with nullptr as an external user must be registered in "(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3221, __extension__ __PRETTY_FUNCTION__))
3221 "ExternallyUsedValues map")(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3221, __extension__ __PRETTY_FUNCTION__));
3222 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3223 Builder.SetInsertPoint(VecI->getParent(),
3224 std::next(VecI->getIterator()));
3225 } else {
3226 Builder.SetInsertPoint(&F->getEntryBlock().front());
3227 }
3228 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3229 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3230 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3231 auto &Locs = ExternallyUsedValues[Scalar];
3232 ExternallyUsedValues.insert({Ex, Locs});
3233 ExternallyUsedValues.erase(Scalar);
3234 continue;
3235 }
3236
3237 // Generate extracts for out-of-tree users.
3238 // Find the insertion point for the extractelement lane.
3239 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3240 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3241 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3242 if (PH->getIncomingValue(i) == Scalar) {
3243 TerminatorInst *IncomingTerminator =
3244 PH->getIncomingBlock(i)->getTerminator();
3245 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3246 Builder.SetInsertPoint(VecI->getParent(),
3247 std::next(VecI->getIterator()));
3248 } else {
3249 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3250 }
3251 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3252 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3253 CSEBlocks.insert(PH->getIncomingBlock(i));
3254 PH->setOperand(i, Ex);
3255 }
3256 }
3257 } else {
3258 Builder.SetInsertPoint(cast<Instruction>(User));
3259 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3260 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3261 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3262 User->replaceUsesOfWith(Scalar, Ex);
3263 }
3264 } else {
3265 Builder.SetInsertPoint(&F->getEntryBlock().front());
3266 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3267 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3268 CSEBlocks.insert(&F->getEntryBlock());
3269 User->replaceUsesOfWith(Scalar, Ex);
3270 }
3271
3272 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false);
3273 }
3274
3275 // For each vectorized value:
3276 for (TreeEntry &EIdx : VectorizableTree) {
3277 TreeEntry *Entry = &EIdx;
3278
3279 // No need to handle users of gathered values.
3280 if (Entry->NeedToGather)
3281 continue;
3282
3283 assert(Entry->VectorizedValue && "Can't find vectorizable value")(static_cast <bool> (Entry->VectorizedValue &&
"Can't find vectorizable value") ? void (0) : __assert_fail (
"Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3283, __extension__ __PRETTY_FUNCTION__));
3284
3285 // For each lane:
3286 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3287 Value *Scalar = Entry->Scalars[Lane];
3288
3289 Type *Ty = Scalar->getType();
3290 if (!Ty->isVoidTy()) {
3291#ifndef NDEBUG
3292 for (User *U : Scalar->users()) {
3293 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false);
3294
3295 // It is legal to replace users in the ignorelist by undef.
3296 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&(static_cast <bool> ((getTreeEntry(U) || is_contained(UserIgnoreList
, U)) && "Replacing out-of-tree value with undef") ? void
(0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3297, __extension__ __PRETTY_FUNCTION__))
3297 "Replacing out-of-tree value with undef")(static_cast <bool> ((getTreeEntry(U) || is_contained(UserIgnoreList
, U)) && "Replacing out-of-tree value with undef") ? void
(0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3297, __extension__ __PRETTY_FUNCTION__));
3298 }
3299#endif
3300 Value *Undef = UndefValue::get(Ty);
3301 Scalar->replaceAllUsesWith(Undef);
3302 }
3303 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false);
3304 eraseInstruction(cast<Instruction>(Scalar));
3305 }
3306 }
3307
3308 Builder.ClearInsertionPoint();
3309
3310 return VectorizableTree[0].VectorizedValue;
3311}
3312
3313void BoUpSLP::optimizeGatherSequence(Function &F) {
3314 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
3315 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false);
3316 // LICM InsertElementInst sequences.
3317 for (Instruction *it : GatherSeq) {
3318 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
3319
3320 if (!Insert)
3321 continue;
3322
3323 // Check if this block is inside a loop.
3324 Loop *L = LI->getLoopFor(Insert->getParent());
3325 if (!L)
3326 continue;
3327
3328 // Check if it has a preheader.
3329 BasicBlock *PreHeader = L->getLoopPreheader();
3330 if (!PreHeader)
3331 continue;
3332
3333 // If the vector or the element that we insert into it are
3334 // instructions that are defined in this basic block then we can't
3335 // hoist this instruction.
3336 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
3337 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
3338 if (CurrVec && L->contains(CurrVec))
3339 continue;
3340 if (NewElem && L->contains(NewElem))
3341 continue;
3342
3343 // We can hoist this instruction. Move it to the pre-header.
3344 Insert->moveBefore(PreHeader->getTerminator());
3345 }
3346
3347 // Perform O(N^2) search over the gather sequences and merge identical
3348 // instructions. TODO: We can further optimize this scan if we split the
3349 // instructions into different buckets based on the insert lane.
3350 SmallVector<Instruction *, 16> Visited;
3351 ReversePostOrderTraversal<Function *> RPOT(&F);
3352 for (auto BB : RPOT) {
3353 // Traverse CSEBlocks by RPOT order.
3354 if (!CSEBlocks.count(BB))
3355 continue;
3356
3357 // For all instructions in blocks containing gather sequences:
3358 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3359 Instruction *In = &*it++;
3360 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3361 continue;
3362
3363 // Check if we can replace this instruction with any of the
3364 // visited instructions.
3365 for (Instruction *v : Visited) {
3366 if (In->isIdenticalTo(v) &&
3367 DT->dominates(v->getParent(), In->getParent())) {
3368 In->replaceAllUsesWith(v);
3369 eraseInstruction(In);
3370 In = nullptr;
3371 break;
3372 }
3373 }
3374 if (In) {
3375 assert(!is_contained(Visited, In))(static_cast <bool> (!is_contained(Visited, In)) ? void
(0) : __assert_fail ("!is_contained(Visited, In)", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3375, __extension__ __PRETTY_FUNCTION__));
3376 Visited.push_back(In);
3377 }
3378 }
3379 }
3380 CSEBlocks.clear();
3381 GatherSeq.clear();
3382}
3383
3384// Groups the instructions to a bundle (which is then a single scheduling entity)
3385// and schedules instructions until the bundle gets ready.
3386bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3387 BoUpSLP *SLP, Value *OpValue) {
3388 if (isa<PHINode>(OpValue))
3389 return true;
3390
3391 // Initialize the instruction bundle.
3392 Instruction *OldScheduleEnd = ScheduleEnd;
3393 ScheduleData *PrevInBundle = nullptr;
3394 ScheduleData *Bundle = nullptr;
3395 bool ReSchedule = false;
3396 DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *OpValue
<< "\n"; } } while (false);
3397
3398 // Make sure that the scheduling region contains all
3399 // instructions of the bundle.
3400 for (Value *V : VL) {
3401 if (!extendSchedulingRegion(V, OpValue))
3402 return false;
3403 }
3404
3405 for (Value *V : VL) {
3406 ScheduleData *BundleMember = getScheduleData(V);
3407 assert(BundleMember &&(static_cast <bool> (BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? void (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3408, __extension__ __PRETTY_FUNCTION__))
3408 "no ScheduleData for bundle member (maybe not in same basic block)")(static_cast <bool> (BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? void (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3408, __extension__ __PRETTY_FUNCTION__));
3409 if (BundleMember->IsScheduled) {
3410 // A bundle member was scheduled as single instruction before and now
3411 // needs to be scheduled as part of the bundle. We just get rid of the
3412 // existing schedule.
3413 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3414 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false);
3415 ReSchedule = true;
3416 }
3417 assert(BundleMember->isSchedulingEntity() &&(static_cast <bool> (BundleMember->isSchedulingEntity
() && "bundle member already part of other bundle") ?
void (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3418, __extension__ __PRETTY_FUNCTION__))
3418 "bundle member already part of other bundle")(static_cast <bool> (BundleMember->isSchedulingEntity
() && "bundle member already part of other bundle") ?
void (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3418, __extension__ __PRETTY_FUNCTION__));
3419 if (PrevInBundle) {
3420 PrevInBundle->NextInBundle = BundleMember;
3421 } else {
3422 Bundle = BundleMember;
3423 }
3424 BundleMember->UnscheduledDepsInBundle = 0;
3425 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3426
3427 // Group the instructions to a bundle.
3428 BundleMember->FirstInBundle = Bundle;
3429 PrevInBundle = BundleMember;
3430 }
3431 if (ScheduleEnd != OldScheduleEnd) {
3432 // The scheduling region got new instructions at the lower end (or it is a
3433 // new region for the first bundle). This makes it necessary to
3434 // recalculate all dependencies.
3435 // It is seldom that this needs to be done a second time after adding the
3436 // initial bundle to the region.
3437 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3438 doForAllOpcodes(I, [](ScheduleData *SD) {
3439 SD->clearDependencies();
3440 });
3441 }
3442 ReSchedule = true;
3443 }
3444 if (ReSchedule) {
3445 resetSchedule();
3446 initialFillReadyList(ReadyInsts);
3447 }
3448
3449 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
3450 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false);
3451
3452 calculateDependencies(Bundle, true, SLP);
3453
3454 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3455 // means that there are no cyclic dependencies and we can schedule it.
3456 // Note that's important that we don't "schedule" the bundle yet (see
3457 // cancelScheduling).
3458 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3459
3460 ScheduleData *pickedSD = ReadyInsts.back();
3461 ReadyInsts.pop_back();
3462
3463 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3464 schedule(pickedSD, ReadyInsts);
3465 }
3466 }
3467 if (!Bundle->isReady()) {
3468 cancelScheduling(VL, OpValue);
3469 return false;
3470 }
3471 return true;
3472}
3473
3474void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3475 Value *OpValue) {
3476 if (isa<PHINode>(OpValue))
3477 return;
3478
3479 ScheduleData *Bundle = getScheduleData(OpValue);
3480 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false);
3481 assert(!Bundle->IsScheduled &&(static_cast <bool> (!Bundle->IsScheduled &&
"Can't cancel bundle which is already scheduled") ? void (0)
: __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3482, __extension__ __PRETTY_FUNCTION__))
3482 "Can't cancel bundle which is already scheduled")(static_cast <bool> (!Bundle->IsScheduled &&
"Can't cancel bundle which is already scheduled") ? void (0)
: __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3482, __extension__ __PRETTY_FUNCTION__));
3483 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&(static_cast <bool> (Bundle->isSchedulingEntity() &&
Bundle->isPartOfBundle() && "tried to unbundle something which is not a bundle"
) ? void (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3484, __extension__ __PRETTY_FUNCTION__))
3484 "tried to unbundle something which is not a bundle")(static_cast <bool> (Bundle->isSchedulingEntity() &&
Bundle->isPartOfBundle() && "tried to unbundle something which is not a bundle"
) ? void (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3484, __extension__ __PRETTY_FUNCTION__));
3485
3486 // Un-bundle: make single instructions out of the bundle.
3487 ScheduleData *BundleMember = Bundle;
3488 while (BundleMember) {
3489 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")(static_cast <bool> (BundleMember->FirstInBundle == Bundle
&& "corrupt bundle links") ? void (0) : __assert_fail
("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3489, __extension__ __PRETTY_FUNCTION__));
3490 BundleMember->FirstInBundle = BundleMember;
3491 ScheduleData *Next = BundleMember->NextInBundle;
3492 BundleMember->NextInBundle = nullptr;
3493 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3494 if (BundleMember->UnscheduledDepsInBundle == 0) {
3495 ReadyInsts.insert(BundleMember);
3496 }
3497 BundleMember = Next;
3498 }
3499}
3500
3501BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3502 // Allocate a new ScheduleData for the instruction.
3503 if (ChunkPos >= ChunkSize) {
3504 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3505 ChunkPos = 0;
3506 }
3507 return &(ScheduleDataChunks.back()[ChunkPos++]);
3508}
3509
3510bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3511 Value *OpValue) {
3512 if (getScheduleData(V, isOneOf(OpValue, V)))
3513 return true;
3514 Instruction *I = dyn_cast<Instruction>(V);
3515 assert(I && "bundle member must be an instruction")(static_cast <bool> (I && "bundle member must be an instruction"
) ? void (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3515, __extension__ __PRETTY_FUNCTION__));
3516 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")(static_cast <bool> (!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? void (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3516, __extension__ __PRETTY_FUNCTION__));
3517 auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3518 ScheduleData *ISD = getScheduleData(I);
3519 if (!ISD)
3520 return false;
3521 assert(isInSchedulingRegion(ISD) &&(static_cast <bool> (isInSchedulingRegion(ISD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3522, __extension__ __PRETTY_FUNCTION__))
3522 "ScheduleData not in scheduling region")(static_cast <bool> (isInSchedulingRegion(ISD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3522, __extension__ __PRETTY_FUNCTION__));
3523 ScheduleData *SD = allocateScheduleDataChunks();
3524 SD->Inst = I;
3525 SD->init(SchedulingRegionID, OpValue);
3526 ExtraScheduleDataMap[I][OpValue] = SD;
3527 return true;
3528 };
3529 if (CheckSheduleForI(I))
3530 return true;
3531 if (!ScheduleStart) {
3532 // It's the first instruction in the new region.
3533 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3534 ScheduleStart = I;
3535 ScheduleEnd = I->getNextNode();
3536 if (isOneOf(OpValue, I) != I)
3537 CheckSheduleForI(I);
3538 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")(static_cast <bool> (ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? void (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3538, __extension__ __PRETTY_FUNCTION__));
3539 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false);
3540 return true;
3541 }
3542 // Search up and down at the same time, because we don't know if the new
3543 // instruction is above or below the existing scheduling region.
3544 BasicBlock::reverse_iterator UpIter =
3545 ++ScheduleStart->getIterator().getReverse();
3546 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3547 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3548 BasicBlock::iterator LowerEnd = BB->end();
3549 while (true) {
3550 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3551 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false);
3552 return false;
3553 }
3554
3555 if (UpIter != UpperEnd) {
3556 if (&*UpIter == I) {
3557 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3558 ScheduleStart = I;
3559 if (isOneOf(OpValue, I) != I)
3560 CheckSheduleForI(I);
3561 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false);
3562 return true;
3563 }
3564 UpIter++;
3565 }
3566 if (DownIter != LowerEnd) {
3567 if (&*DownIter == I) {
3568 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3569 nullptr);
3570 ScheduleEnd = I->getNextNode();
3571 if (isOneOf(OpValue, I) != I)
3572 CheckSheduleForI(I);
3573 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")(static_cast <bool> (ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? void (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3573, __extension__ __PRETTY_FUNCTION__));
3574 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false);
3575 return true;
3576 }
3577 DownIter++;
3578 }
3579 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(static_cast <bool> ((UpIter != UpperEnd || DownIter !=
LowerEnd) && "instruction not found in block") ? void
(0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3580, __extension__ __PRETTY_FUNCTION__))
3580 "instruction not found in block")(static_cast <bool> ((UpIter != UpperEnd || DownIter !=
LowerEnd) && "instruction not found in block") ? void
(0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3580, __extension__ __PRETTY_FUNCTION__));
3581 }
3582 return true;
3583}
3584
3585void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3586 Instruction *ToI,
3587 ScheduleData *PrevLoadStore,
3588 ScheduleData *NextLoadStore) {
3589 ScheduleData *CurrentLoadStore = PrevLoadStore;
3590 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3591 ScheduleData *SD = ScheduleDataMap[I];
3592 if (!SD) {
3593 SD = allocateScheduleDataChunks();
3594 ScheduleDataMap[I] = SD;
3595 SD->Inst = I;
3596 }
3597 assert(!isInSchedulingRegion(SD) &&(static_cast <bool> (!isInSchedulingRegion(SD) &&
"new ScheduleData already in scheduling region") ? void (0) :
__assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3598, __extension__ __PRETTY_FUNCTION__))
3598 "new ScheduleData already in scheduling region")(static_cast <bool> (!isInSchedulingRegion(SD) &&
"new ScheduleData already in scheduling region") ? void (0) :
__assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3598, __extension__ __PRETTY_FUNCTION__));
3599 SD->init(SchedulingRegionID, I);
3600
3601 if (I->mayReadOrWriteMemory() &&
3602 (!isa<IntrinsicInst>(I) ||
3603 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
3604 // Update the linked list of memory accessing instructions.
3605 if (CurrentLoadStore) {
3606 CurrentLoadStore->NextLoadStore = SD;
3607 } else {
3608 FirstLoadStoreInRegion = SD;
3609 }
3610 CurrentLoadStore = SD;
3611 }
3612 }
3613 if (NextLoadStore) {
3614 if (CurrentLoadStore)
3615 CurrentLoadStore->NextLoadStore = NextLoadStore;
3616 } else {
3617 LastLoadStoreInRegion = CurrentLoadStore;
3618 }
3619}
3620
3621void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3622 bool InsertInReadyList,
3623 BoUpSLP *SLP) {
3624 assert(SD->isSchedulingEntity())(static_cast <bool> (SD->isSchedulingEntity()) ? void
(0) : __assert_fail ("SD->isSchedulingEntity()", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3624, __extension__ __PRETTY_FUNCTION__));
3625
3626 SmallVector<ScheduleData *, 10> WorkList;
3627 WorkList.push_back(SD);
3628
3629 while (!WorkList.empty()) {
3630 ScheduleData *SD = WorkList.back();
3631 WorkList.pop_back();
3632
3633 ScheduleData *BundleMember = SD;
3634 while (BundleMember) {
3635 assert(isInSchedulingRegion(BundleMember))(static_cast <bool> (isInSchedulingRegion(BundleMember)
) ? void (0) : __assert_fail ("isInSchedulingRegion(BundleMember)"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3635, __extension__ __PRETTY_FUNCTION__));
3636 if (!BundleMember->hasValidDependencies()) {
3637
3638 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false);
3639 BundleMember->Dependencies = 0;
3640 BundleMember->resetUnscheduledDeps();
3641
3642 // Handle def-use chain dependencies.
3643 if (BundleMember->OpValue != BundleMember->Inst) {
3644 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
3645 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3646 BundleMember->Dependencies++;
3647 ScheduleData *DestBundle = UseSD->FirstInBundle;
3648 if (!DestBundle->IsScheduled)
3649 BundleMember->incrementUnscheduledDeps(1);
3650 if (!DestBundle->hasValidDependencies())
3651 WorkList.push_back(DestBundle);
3652 }
3653 } else {
3654 for (User *U : BundleMember->Inst->users()) {
3655 if (isa<Instruction>(U)) {
3656 ScheduleData *UseSD = getScheduleData(U);
3657 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3658 BundleMember->Dependencies++;
3659 ScheduleData *DestBundle = UseSD->FirstInBundle;
3660 if (!DestBundle->IsScheduled)
3661 BundleMember->incrementUnscheduledDeps(1);
3662 if (!DestBundle->hasValidDependencies())
3663 WorkList.push_back(DestBundle);
3664 }
3665 } else {
3666 // I'm not sure if this can ever happen. But we need to be safe.
3667 // This lets the instruction/bundle never be scheduled and
3668 // eventually disable vectorization.
3669 BundleMember->Dependencies++;
3670 BundleMember->incrementUnscheduledDeps(1);
3671 }
3672 }
3673 }
3674
3675 // Handle the memory dependencies.
3676 ScheduleData *DepDest = BundleMember->NextLoadStore;
3677 if (DepDest) {
3678 Instruction *SrcInst = BundleMember->Inst;
3679 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3680 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3681 unsigned numAliased = 0;
3682 unsigned DistToSrc = 1;
3683
3684 while (DepDest) {
3685 assert(isInSchedulingRegion(DepDest))(static_cast <bool> (isInSchedulingRegion(DepDest)) ? void
(0) : __assert_fail ("isInSchedulingRegion(DepDest)", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3685, __extension__ __PRETTY_FUNCTION__));
3686
3687 // We have two limits to reduce the complexity:
3688 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3689 // SLP->isAliased (which is the expensive part in this loop).
3690 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3691 // the whole loop (even if the loop is fast, it's quadratic).
3692 // It's important for the loop break condition (see below) to
3693 // check this limit even between two read-only instructions.
3694 if (DistToSrc >= MaxMemDepDistance ||
3695 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3696 (numAliased >= AliasedCheckLimit ||
3697 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3698
3699 // We increment the counter only if the locations are aliased
3700 // (instead of counting all alias checks). This gives a better
3701 // balance between reduced runtime and accurate dependencies.
3702 numAliased++;
3703
3704 DepDest->MemoryDependencies.push_back(BundleMember);
3705 BundleMember->Dependencies++;
3706 ScheduleData *DestBundle = DepDest->FirstInBundle;
3707 if (!DestBundle->IsScheduled) {
3708 BundleMember->incrementUnscheduledDeps(1);
3709 }
3710 if (!DestBundle->hasValidDependencies()) {
3711 WorkList.push_back(DestBundle);
3712 }
3713 }
3714 DepDest = DepDest->NextLoadStore;
3715
3716 // Example, explaining the loop break condition: Let's assume our
3717 // starting instruction is i0 and MaxMemDepDistance = 3.
3718 //
3719 // +--------v--v--v
3720 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3721 // +--------^--^--^
3722 //
3723 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3724 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3725 // Previously we already added dependencies from i3 to i6,i7,i8
3726 // (because of MaxMemDepDistance). As we added a dependency from
3727 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3728 // and we can abort this loop at i6.
3729 if (DistToSrc >= 2 * MaxMemDepDistance)
3730 break;
3731 DistToSrc++;
3732 }
3733 }
3734 }
3735 BundleMember = BundleMember->NextInBundle;
3736 }
3737 if (InsertInReadyList && SD->isReady()) {
3738 ReadyInsts.push_back(SD);
3739 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false);
3740 }
3741 }
3742}
3743
3744void BoUpSLP::BlockScheduling::resetSchedule() {
3745 assert(ScheduleStart &&(static_cast <bool> (ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? void (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3746, __extension__ __PRETTY_FUNCTION__))
3746 "tried to reset schedule on block which has not been scheduled")(static_cast <bool> (ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? void (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3746, __extension__ __PRETTY_FUNCTION__));
3747 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3748 doForAllOpcodes(I, [&](ScheduleData *SD) {
3749 assert(isInSchedulingRegion(SD) &&(static_cast <bool> (isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3750, __extension__ __PRETTY_FUNCTION__))
3750 "ScheduleData not in scheduling region")(static_cast <bool> (isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3750, __extension__ __PRETTY_FUNCTION__));
3751 SD->IsScheduled = false;
3752 SD->resetUnscheduledDeps();
3753 });
3754 }
3755 ReadyInsts.clear();
3756}
3757
3758void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3759 if (!BS->ScheduleStart)
3760 return;
3761
3762 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false);
3763
3764 BS->resetSchedule();
3765
3766 // For the real scheduling we use a more sophisticated ready-list: it is
3767 // sorted by the original instruction location. This lets the final schedule
3768 // be as close as possible to the original instruction order.
3769 struct ScheduleDataCompare {
3770 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3771 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3772 }
3773 };
3774 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3775
3776 // Ensure that all dependency data is updated and fill the ready-list with
3777 // initial instructions.
3778 int Idx = 0;
3779 int NumToSchedule = 0;
3780 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3781 I = I->getNextNode()) {
3782 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
3783 assert(SD->isPartOfBundle() ==(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3785, __extension__ __PRETTY_FUNCTION__))
3784 (getTreeEntry(SD->Inst) != nullptr) &&(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3785, __extension__ __PRETTY_FUNCTION__))
3785 "scheduler and vectorizer bundle mismatch")(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3785, __extension__ __PRETTY_FUNCTION__));
3786 SD->FirstInBundle->SchedulingPriority = Idx++;
3787 if (SD->isSchedulingEntity()) {
3788 BS->calculateDependencies(SD, false, this);
3789 NumToSchedule++;
3790 }
3791 });
3792 }
3793 BS->initialFillReadyList(ReadyInsts);
3794
3795 Instruction *LastScheduledInst = BS->ScheduleEnd;
3796
3797 // Do the "real" scheduling.
3798 while (!ReadyInsts.empty()) {
3799 ScheduleData *picked = *ReadyInsts.begin();
3800 ReadyInsts.erase(ReadyInsts.begin());
3801
3802 // Move the scheduled instruction(s) to their dedicated places, if not
3803 // there yet.
3804 ScheduleData *BundleMember = picked;
3805 while (BundleMember) {
3806 Instruction *pickedInst = BundleMember->Inst;
3807 if (LastScheduledInst->getNextNode() != pickedInst) {
3808 BS->BB->getInstList().remove(pickedInst);
3809 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3810 pickedInst);
3811 }
3812 LastScheduledInst = pickedInst;
3813 BundleMember = BundleMember->NextInBundle;
3814 }
3815
3816 BS->schedule(picked, ReadyInsts);
3817 NumToSchedule--;
3818 }
3819 assert(NumToSchedule == 0 && "could not schedule all instructions")(static_cast <bool> (NumToSchedule == 0 && "could not schedule all instructions"
) ? void (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3819, __extension__ __PRETTY_FUNCTION__));
3820
3821 // Avoid duplicate scheduling of the block.
3822 BS->ScheduleStart = nullptr;
3823}
3824
3825unsigned BoUpSLP::getVectorElementSize(Value *V) {
3826 // If V is a store, just return the width of the stored value without
3827 // traversing the expression tree. This is the common case.
3828 if (auto *Store = dyn_cast<StoreInst>(V))
3829 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3830
3831 // If V is not a store, we can traverse the expression tree to find loads
3832 // that feed it. The type of the loaded value may indicate a more suitable
3833 // width than V's type. We want to base the vector element size on the width
3834 // of memory operations where possible.
3835 SmallVector<Instruction *, 16> Worklist;
3836 SmallPtrSet<Instruction *, 16> Visited;
3837 if (auto *I = dyn_cast<Instruction>(V))
3838 Worklist.push_back(I);
3839
3840 // Traverse the expression tree in bottom-up order looking for loads. If we
3841 // encounter an instruciton we don't yet handle, we give up.
3842 auto MaxWidth = 0u;
3843 auto FoundUnknownInst = false;
3844 while (!Worklist.empty() && !FoundUnknownInst) {
3845 auto *I = Worklist.pop_back_val();
3846 Visited.insert(I);
3847
3848 // We should only be looking at scalar instructions here. If the current
3849 // instruction has a vector type, give up.
3850 auto *Ty = I->getType();
3851 if (isa<VectorType>(Ty))
3852 FoundUnknownInst = true;
3853
3854 // If the current instruction is a load, update MaxWidth to reflect the
3855 // width of the loaded value.
3856 else if (isa<LoadInst>(I))
3857 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3858
3859 // Otherwise, we need to visit the operands of the instruction. We only
3860 // handle the interesting cases from buildTree here. If an operand is an
3861 // instruction we haven't yet visited, we add it to the worklist.
3862 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3863 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3864 for (Use &U : I->operands())
3865 if (auto *J = dyn_cast<Instruction>(U.get()))
3866 if (!Visited.count(J))
3867 Worklist.push_back(J);
3868 }
3869
3870 // If we don't yet handle the instruction, give up.
3871 else
3872 FoundUnknownInst = true;
3873 }
3874
3875 // If we didn't encounter a memory access in the expression tree, or if we
3876 // gave up for some reason, just return the width of V.
3877 if (!MaxWidth || FoundUnknownInst)
3878 return DL->getTypeSizeInBits(V->getType());
3879
3880 // Otherwise, return the maximum width we found.
3881 return MaxWidth;
3882}
3883
3884// Determine if a value V in a vectorizable expression Expr can be demoted to a
3885// smaller type with a truncation. We collect the values that will be demoted
3886// in ToDemote and additional roots that require investigating in Roots.
3887static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
3888 SmallVectorImpl<Value *> &ToDemote,
3889 SmallVectorImpl<Value *> &Roots) {
3890 // We can always demote constants.
3891 if (isa<Constant>(V)) {
3892 ToDemote.push_back(V);
3893 return true;
3894 }
3895
3896 // If the value is not an instruction in the expression with only one use, it
3897 // cannot be demoted.
3898 auto *I = dyn_cast<Instruction>(V);
3899 if (!I || !I->hasOneUse() || !Expr.count(I))
3900 return false;
3901
3902 switch (I->getOpcode()) {
3903
3904 // We can always demote truncations and extensions. Since truncations can
3905 // seed additional demotion, we save the truncated value.
3906 case Instruction::Trunc:
3907 Roots.push_back(I->getOperand(0));
3908 break;
3909 case Instruction::ZExt:
3910 case Instruction::SExt:
3911 break;
3912
3913 // We can demote certain binary operations if we can demote both of their
3914 // operands.
3915 case Instruction::Add:
3916 case Instruction::Sub:
3917 case Instruction::Mul:
3918 case Instruction::And:
3919 case Instruction::Or:
3920 case Instruction::Xor:
3921 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3922 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3923 return false;
3924 break;
3925
3926 // We can demote selects if we can demote their true and false values.
3927 case Instruction::Select: {
3928 SelectInst *SI = cast<SelectInst>(I);
3929 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3930 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3931 return false;
3932 break;
3933 }
3934
3935 // We can demote phis if we can demote all their incoming operands. Note that
3936 // we don't need to worry about cycles since we ensure single use above.
3937 case Instruction::PHI: {
3938 PHINode *PN = cast<PHINode>(I);
3939 for (Value *IncValue : PN->incoming_values())
3940 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3941 return false;
3942 break;
3943 }
3944
3945 // Otherwise, conservatively give up.
3946 default:
3947 return false;
3948 }
3949
3950 // Record the value that we can demote.
3951 ToDemote.push_back(V);
3952 return true;
3953}
3954
3955void BoUpSLP::computeMinimumValueSizes() {
3956 // If there are no external uses, the expression tree must be rooted by a
3957 // store. We can't demote in-memory values, so there is nothing to do here.
3958 if (ExternalUses.empty())
3959 return;
3960
3961 // We only attempt to truncate integer expressions.
3962 auto &TreeRoot = VectorizableTree[0].Scalars;
3963 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3964 if (!TreeRootIT)
3965 return;
3966
3967 // If the expression is not rooted by a store, these roots should have
3968 // external uses. We will rely on InstCombine to rewrite the expression in
3969 // the narrower type. However, InstCombine only rewrites single-use values.
3970 // This means that if a tree entry other than a root is used externally, it
3971 // must have multiple uses and InstCombine will not rewrite it. The code
3972 // below ensures that only the roots are used externally.
3973 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3974 for (auto &EU : ExternalUses)
3975 if (!Expr.erase(EU.Scalar))
3976 return;
3977 if (!Expr.empty())
3978 return;
3979
3980 // Collect the scalar values of the vectorizable expression. We will use this
3981 // context to determine which values can be demoted. If we see a truncation,
3982 // we mark it as seeding another demotion.
3983 for (auto &Entry : VectorizableTree)
3984 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3985
3986 // Ensure the roots of the vectorizable tree don't form a cycle. They must
3987 // have a single external user that is not in the vectorizable tree.
3988 for (auto *Root : TreeRoot)
3989 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
3990 return;
3991
3992 // Conservatively determine if we can actually truncate the roots of the
3993 // expression. Collect the values that can be demoted in ToDemote and
3994 // additional roots that require investigating in Roots.
3995 SmallVector<Value *, 32> ToDemote;
3996 SmallVector<Value *, 4> Roots;
3997 for (auto *Root : TreeRoot)
3998 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
3999 return;
4000
4001 // The maximum bit width required to represent all the values that can be
4002 // demoted without loss of precision. It would be safe to truncate the roots
4003 // of the expression to this width.
4004 auto MaxBitWidth = 8u;
4005
4006 // We first check if all the bits of the roots are demanded. If they're not,
4007 // we can truncate the roots to this narrower type.
4008 for (auto *Root : TreeRoot) {
4009 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4010 MaxBitWidth = std::max<unsigned>(
4011 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4012 }
4013
4014 // True if the roots can be zero-extended back to their original type, rather
4015 // than sign-extended. We know that if the leading bits are not demanded, we
4016 // can safely zero-extend. So we initialize IsKnownPositive to True.
4017 bool IsKnownPositive = true;
4018
4019 // If all the bits of the roots are demanded, we can try a little harder to
4020 // compute a narrower type. This can happen, for example, if the roots are
4021 // getelementptr indices. InstCombine promotes these indices to the pointer
4022 // width. Thus, all their bits are technically demanded even though the
4023 // address computation might be vectorized in a smaller type.
4024 //
4025 // We start by looking at each entry that can be demoted. We compute the
4026 // maximum bit width required to store the scalar by using ValueTracking to
4027 // compute the number of high-order bits we can truncate.
4028 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
4029 MaxBitWidth = 8u;
4030
4031 // Determine if the sign bit of all the roots is known to be zero. If not,
4032 // IsKnownPositive is set to False.
4033 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4034 KnownBits Known = computeKnownBits(R, *DL);
4035 return Known.isNonNegative();
4036 });
4037
4038 // Determine the maximum number of bits required to store the scalar
4039 // values.
4040 for (auto *Scalar : ToDemote) {
4041 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4042 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4043 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4044 }
4045
4046 // If we can't prove that the sign bit is zero, we must add one to the
4047 // maximum bit width to account for the unknown sign bit. This preserves
4048 // the existing sign bit so we can safely sign-extend the root back to the
4049 // original type. Otherwise, if we know the sign bit is zero, we will
4050 // zero-extend the root instead.
4051 //
4052 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4053 // one to the maximum bit width will yield a larger-than-necessary
4054 // type. In general, we need to add an extra bit only if we can't
4055 // prove that the upper bit of the original type is equal to the
4056 // upper bit of the proposed smaller type. If these two bits are the
4057 // same (either zero or one) we know that sign-extending from the
4058 // smaller type will result in the same value. Here, since we can't
4059 // yet prove this, we are just making the proposed smaller type
4060 // larger to ensure correctness.
4061 if (!IsKnownPositive)
4062 ++MaxBitWidth;
4063 }
4064
4065 // Round MaxBitWidth up to the next power-of-two.
4066 if (!isPowerOf2_64(MaxBitWidth))
4067 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4068
4069 // If the maximum bit width we compute is less than the with of the roots'
4070 // type, we can proceed with the narrowing. Otherwise, do nothing.
4071 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4072 return;
4073
4074 // If we can truncate the root, we must collect additional values that might
4075 // be demoted as a result. That is, those seeded by truncations we will
4076 // modify.
4077 while (!Roots.empty())
4078 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4079
4080 // Finally, map the values we can demote to the maximum bit with we computed.
4081 for (auto *Scalar : ToDemote)
4082 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4083}
4084
4085namespace {
4086
4087/// The SLPVectorizer Pass.
4088struct SLPVectorizer : public FunctionPass {
4089 SLPVectorizerPass Impl;
4090
4091 /// Pass identification, replacement for typeid
4092 static char ID;
4093
4094 explicit SLPVectorizer() : FunctionPass(ID) {
4095 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4096 }
4097
4098 bool doInitialization(Module &M) override {
4099 return false;
4100 }
4101
4102 bool runOnFunction(Function &F) override {
4103 if (skipFunction(F))
4104 return false;
4105
4106 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4107 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4108 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4109 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4110 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4111 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4112 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4113 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4114 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4115 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4116
4117 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4118 }
4119
4120 void getAnalysisUsage(AnalysisUsage &AU) const override {
4121 FunctionPass::getAnalysisUsage(AU);
4122 AU.addRequired<AssumptionCacheTracker>();
4123 AU.addRequired<ScalarEvolutionWrapperPass>();
4124 AU.addRequired<AAResultsWrapperPass>();
4125 AU.addRequired<TargetTransformInfoWrapperPass>();
4126 AU.addRequired<LoopInfoWrapperPass>();
4127 AU.addRequired<DominatorTreeWrapperPass>();
4128 AU.addRequired<DemandedBitsWrapperPass>();
4129 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4130 AU.addPreserved<LoopInfoWrapperPass>();
4131 AU.addPreserved<DominatorTreeWrapperPass>();
4132 AU.addPreserved<AAResultsWrapperPass>();
4133 AU.addPreserved<GlobalsAAWrapperPass>();
4134 AU.setPreservesCFG();
4135 }
4136};
4137
4138} // end anonymous namespace
4139
4140PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4141 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4142 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4143 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4144 auto *AA = &AM.getResult<AAManager>(F);
4145 auto *LI = &AM.getResult<LoopAnalysis>(F);
4146 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4147 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4148 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4149 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4150
4151 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4152 if (!Changed)
4153 return PreservedAnalyses::all();
4154
4155 PreservedAnalyses PA;
4156 PA.preserveSet<CFGAnalyses>();
4157 PA.preserve<AAManager>();
4158 PA.preserve<GlobalsAA>();
4159 return PA;
4160}
4161
4162bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4163 TargetTransformInfo *TTI_,
4164 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4165 LoopInfo *LI_, DominatorTree *DT_,
4166 AssumptionCache *AC_, DemandedBits *DB_,
4167 OptimizationRemarkEmitter *ORE_) {
4168 SE = SE_;
4169 TTI = TTI_;
4170 TLI = TLI_;
4171 AA = AA_;
4172 LI = LI_;
4173 DT = DT_;
4174 AC = AC_;
4175 DB = DB_;
4176 DL = &F.getParent()->getDataLayout();
4177
4178 Stores.clear();
4179 GEPs.clear();
4180 bool Changed = false;
4181
4182 // If the target claims to have no vector registers don't attempt
4183 // vectorization.
4184 if (!TTI->getNumberOfRegisters(true))
4185 return false;
4186
4187 // Don't vectorize when the attribute NoImplicitFloat is used.
4188 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4189 return false;
4190
4191 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false);
4192
4193 // Use the bottom up slp vectorizer to construct chains that start with
4194 // store instructions.
4195 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4196
4197 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4198 // delete instructions.
4199
4200 // Scan the blocks in the function in post order.
4201 for (auto BB : post_order(&F.getEntryBlock())) {
4202 collectSeedInstructions(BB);
4203
4204 // Vectorize trees that end at stores.
4205 if (!Stores.empty()) {
4206 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
4207 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false);
4208 Changed |= vectorizeStoreChains(R);
4209 }
4210
4211 // Vectorize trees that end at reductions.
4212 Changed |= vectorizeChainsInBlock(BB, R);
4213
4214 // Vectorize the index computations of getelementptr instructions. This
4215 // is primarily intended to catch gather-like idioms ending at
4216 // non-consecutive loads.
4217 if (!GEPs.empty()) {
4218 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
4219 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false);
4220 Changed |= vectorizeGEPIndices(BB, R);
4221 }
4222 }
4223
4224 if (Changed) {
4225 R.optimizeGatherSequence(F);
4226 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false);
4227 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false);
4228 }
4229 return Changed;
4230}
4231
4232/// \brief Check that the Values in the slice in VL array are still existent in
4233/// the WeakTrackingVH array.
4234/// Vectorization of part of the VL array may cause later values in the VL array
4235/// to become invalid. We track when this has happened in the WeakTrackingVH
4236/// array.
4237static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4238 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4239 unsigned SliceSize) {
4240 VL = VL.slice(SliceBegin, SliceSize);
4241 VH = VH.slice(SliceBegin, SliceSize);
4242 return !std::equal(VL.begin(), VL.end(), VH.begin());
4243}
4244
4245bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4246 unsigned VecRegSize) {
4247 unsigned ChainLen = Chain.size();
4248 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
4249 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false);
4250 unsigned Sz = R.getVectorElementSize(Chain[0]);
4251 unsigned VF = VecRegSize / Sz;
4252
4253 if (!isPowerOf2_32(Sz) || VF < 2)
4254 return false;
4255
4256 // Keep track of values that were deleted by vectorizing in the loop below.
4257 SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4258
4259 bool Changed = false;
4260 // Look for profitable vectorizable trees at all offsets, starting at zero.
4261 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
4262 if (i + VF > e)
4263 break;
4264
4265 // Check that a previous iteration of this loop did not delete the Value.
4266 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4267 continue;
4268
4269 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
4270 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
);
4271 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4272
4273 R.buildTree(Operands);
4274 if (R.isTreeTinyAndNotFullyVectorizable())
4275 continue;
4276
4277 R.computeMinimumValueSizes();
4278
4279 int Cost = R.getTreeCost();
4280
4281 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false);
4282 if (Cost < -SLPCostThreshold) {
4283 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false);
4284
4285 using namespace ore;
4286
4287 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "StoresVectorized",
4288 cast<StoreInst>(Chain[i]))
4289 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4290 << " and with tree size "
4291 << NV("TreeSize", R.getTreeSize()));
4292
4293 R.vectorizeTree();
4294
4295 // Move to the next bundle.
4296 i += VF - 1;
4297 Changed = true;
4298 }
4299 }
4300
4301 return Changed;
4302}
4303
4304bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4305 BoUpSLP &R) {
4306 SetVector<StoreInst *> Heads;
4307 SmallDenseSet<StoreInst *> Tails;
4308 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4309
4310 // We may run into multiple chains that merge into a single chain. We mark the
4311 // stores that we vectorized so that we don't visit the same store twice.
4312 BoUpSLP::ValueSet VectorizedStores;
4313 bool Changed = false;
4314
4315 // Do a quadratic search on all of the given stores in reverse order and find
4316 // all of the pairs of stores that follow each other.
4317 SmallVector<unsigned, 16> IndexQueue;
4318 unsigned E = Stores.size();
4319 IndexQueue.resize(E - 1);
4320 for (unsigned I = E; I > 0; --I) {
4321 unsigned Idx = I - 1;
4322 // If a store has multiple consecutive store candidates, search Stores
4323 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4324 // This is because usually pairing with immediate succeeding or preceding
4325 // candidate create the best chance to find slp vectorization opportunity.
4326 unsigned Offset = 1;
4327 unsigned Cnt = 0;
4328 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4329 if (Idx >= Offset) {
4330 IndexQueue[Cnt] = Idx - Offset;
4331 ++Cnt;
4332 }
4333 if (Idx + Offset < E) {
4334 IndexQueue[Cnt] = Idx + Offset;
4335 ++Cnt;
4336 }
4337 }
4338
4339 for (auto K : IndexQueue) {
4340 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4341 Tails.insert(Stores[Idx]);
4342 Heads.insert(Stores[K]);
4343 ConsecutiveChain[Stores[K]] = Stores[Idx];
4344 break;
4345 }
4346 }
4347 }
4348
4349 // For stores that start but don't end a link in the chain:
4350 for (auto *SI : llvm::reverse(Heads)) {
4351 if (Tails.count(SI))
4352 continue;
4353
4354 // We found a store instr that starts a chain. Now follow the chain and try
4355 // to vectorize it.
4356 BoUpSLP::ValueList Operands;
4357 StoreInst *I = SI;
4358 // Collect the chain into a list.
4359 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4360 Operands.push_back(I);
4361 // Move to the next value in the chain.
4362 I = ConsecutiveChain[I];
4363 }
4364
4365 // FIXME: Is division-by-2 the correct step? Should we assert that the
4366 // register size is a power-of-2?
4367 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4368 Size /= 2) {
4369 if (vectorizeStoreChain(Operands, R, Size)) {
4370 // Mark the vectorized stores so that we don't vectorize them again.
4371 VectorizedStores.insert(Operands.begin(), Operands.end());
4372 Changed = true;
4373 break;
4374 }
4375 }
4376 }
4377
4378 return Changed;
4379}
4380
4381void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4382 // Initialize the collections. We will make a single pass over the block.
4383 Stores.clear();
4384 GEPs.clear();
4385
4386 // Visit the store and getelementptr instructions in BB and organize them in
4387 // Stores and GEPs according to the underlying objects of their pointer
4388 // operands.
4389 for (Instruction &I : *BB) {
4390 // Ignore store instructions that are volatile or have a pointer operand
4391 // that doesn't point to a scalar type.
4392 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4393 if (!SI->isSimple())
4394 continue;
4395 if (!isValidElementType(SI->getValueOperand()->getType()))
4396 continue;
4397 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4398 }
4399
4400 // Ignore getelementptr instructions that have more than one index, a
4401 // constant index, or a pointer operand that doesn't point to a scalar
4402 // type.
4403 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4404 auto Idx = GEP->idx_begin()->get();
4405 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4406 continue;
4407 if (!isValidElementType(Idx->getType()))
4408 continue;
4409 if (GEP->getType()->isVectorTy())
4410 continue;
4411 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4412 }
4413 }
4414}
4415
4416bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4417 if (!A || !B)
4418 return false;
4419 Value *VL[] = { A, B };
4420 return tryToVectorizeList(VL, R, None, true);
4421}
4422
4423bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4424 ArrayRef<Value *> BuildVector,
4425 bool AllowReorder,
4426 bool NeedExtraction) {
4427 if (VL.size() < 2)
4428 return false;
4429
4430 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4431 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false);
4432
4433 // Check that all of the parts are scalar instructions of the same type.
4434 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4435 if (!I0)
4436 return false;
4437
4438 unsigned Opcode0 = I0->getOpcode();
4439
4440 unsigned Sz = R.getVectorElementSize(I0);
4441 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4442 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4443 if (MaxVF < 2) {
4444 R.getORE()->emit([&]() {
4445 return OptimizationRemarkMissed(
4446 SV_NAME"slp-vectorizer", "SmallVF", I0)
4447 << "Cannot SLP vectorize list: vectorization factor "
4448 << "less than 2 is not supported";
4449 });
4450 return false;
4451 }
4452
4453 for (Value *V : VL) {
4454 Type *Ty = V->getType();
4455 if (!isValidElementType(Ty)) {
4456 // NOTE: the following will give user internal llvm type name, which may not be useful
4457 R.getORE()->emit([&]() {
4458 std::string type_str;
4459 llvm::raw_string_ostream rso(type_str);
4460 Ty->print(rso);
4461 return OptimizationRemarkMissed(
4462 SV_NAME"slp-vectorizer", "UnsupportedType", I0)
4463 << "Cannot SLP vectorize list: type "
4464 << rso.str() + " is unsupported by vectorizer";
4465 });
4466 return false;
4467 }
4468 Instruction *Inst = dyn_cast<Instruction>(V);
4469
4470 if (!Inst)
4471 return false;
4472 if (Inst->getOpcode() != Opcode0) {
4473 R.getORE()->emit([&]() {
4474 return OptimizationRemarkMissed(
4475 SV_NAME"slp-vectorizer", "InequableTypes", I0)
4476 << "Cannot SLP vectorize list: not all of the "
4477 << "parts of scalar instructions are of the same type: "
4478 << ore::NV("Instruction1Opcode", I0) << " and "
4479 << ore::NV("Instruction2Opcode", Inst);
4480 });
4481 return false;
4482 }
4483 }
4484
4485 bool Changed = false;
4486 bool CandidateFound = false;
4487 int MinCost = SLPCostThreshold;
4488
4489 // Keep track of values that were deleted by vectorizing in the loop below.
4490 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4491
4492 unsigned NextInst = 0, MaxInst = VL.size();
4493 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4494 VF /= 2) {
4495 // No actual vectorization should happen, if number of parts is the same as
4496 // provided vectorization factor (i.e. the scalar type is used for vector
4497 // code during codegen).
4498 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4499 if (TTI->getNumberOfParts(VecTy) == VF)
4500 continue;
4501 for (unsigned I = NextInst; I < MaxInst; ++I) {
4502 unsigned OpsWidth = 0;
4503
4504 if (I + VF > MaxInst)
4505 OpsWidth = MaxInst - I;
4506 else
4507 OpsWidth = VF;
4508
4509 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4510 break;
4511
4512 // Check that a previous iteration of this loop did not delete the Value.
4513 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4514 continue;
4515
4516 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4517 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false);
4518 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4519
4520 ArrayRef<Value *> EmptyArray;
4521 ArrayRef<Value *> BuildVectorSlice;
4522 if (!BuildVector.empty())
4523 BuildVectorSlice = BuildVector.slice(I, OpsWidth);
4524
4525 R.buildTree(Ops, NeedExtraction ? EmptyArray : BuildVectorSlice);
4526 // TODO: check if we can allow reordering for more cases.
4527 if (AllowReorder && R.shouldReorder()) {
4528 // Conceptually, there is nothing actually preventing us from trying to
4529 // reorder a larger list. In fact, we do exactly this when vectorizing
4530 // reductions. However, at this point, we only expect to get here when
4531 // there are exactly two operations.
4532 assert(Ops.size() == 2)(static_cast <bool> (Ops.size() == 2) ? void (0) : __assert_fail
("Ops.size() == 2", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4532, __extension__ __PRETTY_FUNCTION__));
4533 assert(BuildVectorSlice.empty())(static_cast <bool> (BuildVectorSlice.empty()) ? void (
0) : __assert_fail ("BuildVectorSlice.empty()", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4533, __extension__ __PRETTY_FUNCTION__));
4534 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4535 R.buildTree(ReorderedOps, None);
4536 }
4537 if (R.isTreeTinyAndNotFullyVectorizable())
4538 continue;
4539
4540 R.computeMinimumValueSizes();
4541 int Cost = R.getTreeCost();
4542 CandidateFound = true;
4543 MinCost = std::min(MinCost, Cost);
4544
4545 if (Cost < -SLPCostThreshold) {
4546 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false);
4547 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedList",
4548 cast<Instruction>(Ops[0]))
4549 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4550 << " and with tree size "
4551 << ore::NV("TreeSize", R.getTreeSize()));
4552
4553 Value *VectorizedRoot = R.vectorizeTree();
4554
4555 // Reconstruct the build vector by extracting the vectorized root. This
4556 // way we handle the case where some elements of the vector are
4557 // undefined.
4558 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
4559 if (!BuildVectorSlice.empty()) {
4560 // The insert point is the last build vector instruction. The
4561 // vectorized root will precede it. This guarantees that we get an
4562 // instruction. The vectorized tree could have been constant folded.
4563 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4564 unsigned VecIdx = 0;
4565 for (auto &V : BuildVectorSlice) {
4566 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4567 ++BasicBlock::iterator(InsertAfter));
4568 Instruction *I = cast<Instruction>(V);
4569 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I))(static_cast <bool> (isa<InsertElementInst>(I) ||
isa<InsertValueInst>(I)) ? void (0) : __assert_fail ("isa<InsertElementInst>(I) || isa<InsertValueInst>(I)"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4569, __extension__ __PRETTY_FUNCTION__));
4570 Instruction *Extract =
4571 cast<Instruction>(Builder.CreateExtractElement(
4572 VectorizedRoot, Builder.getInt32(VecIdx++)));
4573 I->setOperand(1, Extract);
4574 I->moveAfter(Extract);
4575 InsertAfter = I;
4576 }
4577 }
4578 // Move to the next bundle.
4579 I += VF - 1;
4580 NextInst = I + 1;
4581 Changed = true;
4582 }
4583 }
4584 }
4585
4586 if (!Changed && CandidateFound) {
4587 R.getORE()->emit([&]() {
4588 return OptimizationRemarkMissed(
4589 SV_NAME"slp-vectorizer", "NotBeneficial", I0)
4590 << "List vectorization was possible but not beneficial with cost "
4591 << ore::NV("Cost", MinCost) << " >= "
4592 << ore::NV("Treshold", -SLPCostThreshold);
4593 });
4594 } else if (!Changed) {
4595 R.getORE()->emit([&]() {
4596 return OptimizationRemarkMissed(
4597 SV_NAME"slp-vectorizer", "NotPossible", I0)
4598 << "Cannot SLP vectorize list: vectorization was impossible"
4599 << " with available vectorization factors";
4600 });
4601 }
4602 return Changed;
4603}
4604
4605bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4606 if (!I)
4607 return false;
4608
4609 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4610 return false;
4611
4612 Value *P = I->getParent();
4613
4614 // Vectorize in current basic block only.
4615 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4616 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4617 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4618 return false;
4619
4620 // Try to vectorize V.
4621 if (tryToVectorizePair(Op0, Op1, R))
4622 return true;
4623
4624 auto *A = dyn_cast<BinaryOperator>(Op0);
4625 auto *B = dyn_cast<BinaryOperator>(Op1);
4626 // Try to skip B.
4627 if (B && B->hasOneUse()) {
4628 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4629 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4630 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4631 return true;
4632 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4633 return true;
4634 }
4635
4636 // Try to skip A.
4637 if (A && A->hasOneUse()) {
4638 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4639 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4640 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4641 return true;
4642 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4643 return true;
4644 }
4645 return false;
4646}
4647
4648/// \brief Generate a shuffle mask to be used in a reduction tree.
4649///
4650/// \param VecLen The length of the vector to be reduced.
4651/// \param NumEltsToRdx The number of elements that should be reduced in the
4652/// vector.
4653/// \param IsPairwise Whether the reduction is a pairwise or splitting
4654/// reduction. A pairwise reduction will generate a mask of
4655/// <0,2,...> or <1,3,..> while a splitting reduction will generate
4656/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4657/// \param IsLeft True will generate a mask of even elements, odd otherwise.
4658static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4659 bool IsPairwise, bool IsLeft,
4660 IRBuilder<> &Builder) {
4661 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(static_cast <bool> ((IsPairwise || !IsLeft) &&
"Don't support a <0,1,undef,...> mask") ? void (0) : __assert_fail
("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4661, __extension__ __PRETTY_FUNCTION__));
4662
4663 SmallVector<Constant *, 32> ShuffleMask(
4664 VecLen, UndefValue::get(Builder.getInt32Ty()));
4665
4666 if (IsPairwise)
4667 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4668 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4669 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4670 else
4671 // Move the upper half of the vector to the lower half.
4672 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4673 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4674
4675 return ConstantVector::get(ShuffleMask);
4676}
4677
4678namespace {
4679
4680/// Model horizontal reductions.
4681///
4682/// A horizontal reduction is a tree of reduction operations (currently add and
4683/// fadd) that has operations that can be put into a vector as its leaf.
4684/// For example, this tree:
4685///
4686/// mul mul mul mul
4687/// \ / \ /
4688/// + +
4689/// \ /
4690/// +
4691/// This tree has "mul" as its reduced values and "+" as its reduction
4692/// operations. A reduction might be feeding into a store or a binary operation
4693/// feeding a phi.
4694/// ...
4695/// \ /
4696/// +
4697/// |
4698/// phi +=
4699///
4700/// Or:
4701/// ...
4702/// \ /
4703/// +
4704/// |
4705/// *p =
4706///
4707class HorizontalReduction {
4708 using ReductionOpsType = SmallVector<Value *, 16>;
4709 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
4710 ReductionOpsListType ReductionOps;
4711 SmallVector<Value *, 32> ReducedVals;
4712 // Use map vector to make stable output.
4713 MapVector<Instruction *, Value *> ExtraArgs;
4714
4715 /// Kind of the reduction data.
4716 enum ReductionKind {
4717 RK_None, /// Not a reduction.
4718 RK_Arithmetic, /// Binary reduction data.
4719 RK_Min, /// Minimum reduction data.
4720 RK_UMin, /// Unsigned minimum reduction data.
4721 RK_Max, /// Maximum reduction data.
4722 RK_UMax, /// Unsigned maximum reduction data.
4723 };
4724
4725 /// Contains info about operation, like its opcode, left and right operands.
4726 class OperationData {
4727 /// Opcode of the instruction.
4728 unsigned Opcode = 0;
4729
4730 /// Left operand of the reduction operation.
4731 Value *LHS = nullptr;
4732
4733 /// Right operand of the reduction operation.
4734 Value *RHS = nullptr;
4735
4736 /// Kind of the reduction operation.
4737 ReductionKind Kind = RK_None;
4738
4739 /// True if float point min/max reduction has no NaNs.
4740 bool NoNaN = false;
4741
4742 /// Checks if the reduction operation can be vectorized.
4743 bool isVectorizable() const {
4744 return LHS && RHS &&
4745 // We currently only support adds && min/max reductions.
4746 ((Kind == RK_Arithmetic &&
4747 (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
4748 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
4749 (Kind == RK_Min || Kind == RK_Max)) ||
4750 (Opcode == Instruction::ICmp &&
4751 (Kind == RK_UMin || Kind == RK_UMax)));
4752 }
4753
4754 /// Creates reduction operation with the current opcode.
4755 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
4756 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4757, __extension__ __PRETTY_FUNCTION__))
4757 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4757, __extension__ __PRETTY_FUNCTION__));
4758 Value *Cmp;
4759 switch (Kind) {
4760 case RK_Arithmetic:
4761 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
4762 Name);
4763 case RK_Min:
4764 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
4765 : Builder.CreateFCmpOLT(LHS, RHS);
4766 break;
4767 case RK_Max:
4768 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
4769 : Builder.CreateFCmpOGT(LHS, RHS);
4770 break;
4771 case RK_UMin:
4772 assert(Opcode == Instruction::ICmp && "Expected integer types.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Expected integer types.") ? void (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4772, __extension__ __PRETTY_FUNCTION__));
4773 Cmp = Builder.CreateICmpULT(LHS, RHS);
4774 break;
4775 case RK_UMax:
4776 assert(Opcode == Instruction::ICmp && "Expected integer types.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Expected integer types.") ? void (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4776, __extension__ __PRETTY_FUNCTION__));
4777 Cmp = Builder.CreateICmpUGT(LHS, RHS);
4778 break;
4779 case RK_None:
4780 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4780);
4781 }
4782 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
4783 }
4784
4785 public:
4786 explicit OperationData() = default;
4787
4788 /// Construction for reduced values. They are identified by opcode only and
4789 /// don't have associated LHS/RHS values.
4790 explicit OperationData(Value *V) {
4791 if (auto *I = dyn_cast<Instruction>(V))
4792 Opcode = I->getOpcode();
4793 }
4794
4795 /// Constructor for reduction operations with opcode and its left and
4796 /// right operands.
4797 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
4798 bool NoNaN = false)
4799 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
4800 assert(Kind != RK_None && "One of the reduction operations is expected.")(static_cast <bool> (Kind != RK_None && "One of the reduction operations is expected."
) ? void (0) : __assert_fail ("Kind != RK_None && \"One of the reduction operations is expected.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4800, __extension__ __PRETTY_FUNCTION__));
4801 }
4802
4803 explicit operator bool() const { return Opcode; }
4804
4805 /// Get the index of the first operand.
4806 unsigned getFirstOperandIndex() const {
4807 assert(!!*this && "The opcode is not set.")(static_cast <bool> (!!*this && "The opcode is not set."
) ? void (0) : __assert_fail ("!!*this && \"The opcode is not set.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4807, __extension__ __PRETTY_FUNCTION__));
4808 switch (Kind) {
4809 case RK_Min:
4810 case RK_UMin:
4811 case RK_Max:
4812 case RK_UMax:
4813 return 1;
4814 case RK_Arithmetic:
4815 case RK_None:
4816 break;
4817 }
4818 return 0;
4819 }
4820
4821 /// Total number of operands in the reduction operation.
4822 unsigned getNumberOfOperands() const {
4823 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4824, __extension__ __PRETTY_FUNCTION__))
4824 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4824, __extension__ __PRETTY_FUNCTION__));
4825 switch (Kind) {
4826 case RK_Arithmetic:
4827 return 2;
4828 case RK_Min:
4829 case RK_UMin:
4830 case RK_Max:
4831 case RK_UMax:
4832 return 3;
4833 case RK_None:
4834 break;
4835 }
4836 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4836);
4837 }
4838
4839 /// Checks if the operation has the same parent as \p P.
4840 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
4841 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4842, __extension__ __PRETTY_FUNCTION__))
4842 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4842, __extension__ __PRETTY_FUNCTION__));
4843 if (!IsRedOp)
4844 return I->getParent() == P;
4845 switch (Kind) {
4846 case RK_Arithmetic:
4847 // Arithmetic reduction operation must be used once only.
4848 return I->getParent() == P;
4849 case RK_Min:
4850 case RK_UMin:
4851 case RK_Max:
4852 case RK_UMax: {
4853 // SelectInst must be used twice while the condition op must have single
4854 // use only.
4855 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
4856 return I->getParent() == P && Cmp && Cmp->getParent() == P;
4857 }
4858 case RK_None:
4859 break;
4860 }
4861 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4861);
4862 }
4863 /// Expected number of uses for reduction operations/reduced values.
4864 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
4865 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4866, __extension__ __PRETTY_FUNCTION__))
4866 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4866, __extension__ __PRETTY_FUNCTION__));
4867 switch (Kind) {
4868 case RK_Arithmetic:
4869 return I->hasOneUse();
4870 case RK_Min:
4871 case RK_UMin:
4872 case RK_Max:
4873 case RK_UMax:
4874 return I->hasNUses(2) &&
4875 (!IsReductionOp ||
4876 cast<SelectInst>(I)->getCondition()->hasOneUse());
4877 case RK_None:
4878 break;
4879 }
4880 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4880);
4881 }
4882
4883 /// Initializes the list of reduction operations.
4884 void initReductionOps(ReductionOpsListType &ReductionOps) {
4885 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4886, __extension__ __PRETTY_FUNCTION__))
4886 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4886, __extension__ __PRETTY_FUNCTION__));
4887 switch (Kind) {
4888 case RK_Arithmetic:
4889 ReductionOps.assign(1, ReductionOpsType());
4890 break;
4891 case RK_Min:
4892 case RK_UMin:
4893 case RK_Max:
4894 case RK_UMax:
4895 ReductionOps.assign(2, ReductionOpsType());
4896 break;
4897 case RK_None:
4898 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4898);
4899 }
4900 }
4901 /// Add all reduction operations for the reduction instruction \p I.
4902 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
4903 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4904, __extension__ __PRETTY_FUNCTION__))
4904 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4904, __extension__ __PRETTY_FUNCTION__));
4905 switch (Kind) {
4906 case RK_Arithmetic:
4907 ReductionOps[0].emplace_back(I);
4908 break;
4909 case RK_Min:
4910 case RK_UMin:
4911 case RK_Max:
4912 case RK_UMax:
4913 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
4914 ReductionOps[1].emplace_back(I);
4915 break;
4916 case RK_None:
4917 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4917);
4918 }
4919 }
4920
4921 /// Checks if instruction is associative and can be vectorized.
4922 bool isAssociative(Instruction *I) const {
4923 assert(Kind != RK_None && *this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && *this &&
LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4924, __extension__ __PRETTY_FUNCTION__))
4924 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && *this &&
LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4924, __extension__ __PRETTY_FUNCTION__));
4925 switch (Kind) {
4926 case RK_Arithmetic:
4927 return I->isAssociative();
4928 case RK_Min:
4929 case RK_Max:
4930 return Opcode == Instruction::ICmp ||
4931 cast<Instruction>(I->getOperand(0))->isFast();
4932 case RK_UMin:
4933 case RK_UMax:
4934 assert(Opcode == Instruction::ICmp &&(static_cast <bool> (Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.") ? void (0) : __assert_fail
("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4935, __extension__ __PRETTY_FUNCTION__))
4935 "Only integer compare operation is expected.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.") ? void (0) : __assert_fail
("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4935, __extension__ __PRETTY_FUNCTION__));
4936 return true;
4937 case RK_None:
4938 break;
4939 }
4940 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4940);
4941 }
4942
4943 /// Checks if the reduction operation can be vectorized.
4944 bool isVectorizable(Instruction *I) const {
4945 return isVectorizable() && isAssociative(I);
4946 }
4947
4948 /// Checks if two operation data are both a reduction op or both a reduced
4949 /// value.
4950 bool operator==(const OperationData &OD) {
4951 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&(static_cast <bool> (((Kind != OD.Kind) || ((!LHS == !OD
.LHS) && (!RHS == !OD.RHS))) && "One of the comparing operations is incorrect."
) ? void (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4952, __extension__ __PRETTY_FUNCTION__))
4952 "One of the comparing operations is incorrect.")(static_cast <bool> (((Kind != OD.Kind) || ((!LHS == !OD
.LHS) && (!RHS == !OD.RHS))) && "One of the comparing operations is incorrect."
) ? void (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4952, __extension__ __PRETTY_FUNCTION__));
4953 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
4954 }
4955 bool operator!=(const OperationData &OD) { return !(*this == OD); }
4956 void clear() {
4957 Opcode = 0;
4958 LHS = nullptr;
4959 RHS = nullptr;
4960 Kind = RK_None;
4961 NoNaN = false;
4962 }
4963
4964 /// Get the opcode of the reduction operation.
4965 unsigned getOpcode() const {
4966 assert(isVectorizable() && "Expected vectorizable operation.")(static_cast <bool> (isVectorizable() && "Expected vectorizable operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected vectorizable operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4966, __extension__ __PRETTY_FUNCTION__));
4967 return Opcode;
4968 }
4969
4970 /// Get kind of reduction data.
4971 ReductionKind getKind() const { return Kind; }
4972 Value *getLHS() const { return LHS; }
4973 Value *getRHS() const { return RHS; }
4974 Type *getConditionType() const {
4975 switch (Kind) {
4976 case RK_Arithmetic:
4977 return nullptr;
4978 case RK_Min:
4979 case RK_Max:
4980 case RK_UMin:
4981 case RK_UMax:
4982 return CmpInst::makeCmpResultType(LHS->getType());
4983 case RK_None:
4984 break;
4985 }
4986 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4986);
4987 }
4988
4989 /// Creates reduction operation with the current opcode with the IR flags
4990 /// from \p ReductionOps.
4991 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
4992 const ReductionOpsListType &ReductionOps) const {
4993 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4994, __extension__ __PRETTY_FUNCTION__))
4994 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4994, __extension__ __PRETTY_FUNCTION__));
4995 auto *Op = createOp(Builder, Name);
4996 switch (Kind) {
4997 case RK_Arithmetic:
4998 propagateIRFlags(Op, ReductionOps[0]);
4999 return Op;
5000 case RK_Min:
5001 case RK_Max:
5002 case RK_UMin:
5003 case RK_UMax:
5004 if (auto *SI = dyn_cast<SelectInst>(Op))
5005 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5006 propagateIRFlags(Op, ReductionOps[1]);
5007 return Op;
5008 case RK_None:
5009 break;
5010 }
5011 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5011);
5012 }
5013 /// Creates reduction operation with the current opcode with the IR flags
5014 /// from \p I.
5015 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5016 Instruction *I) const {
5017 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5018, __extension__ __PRETTY_FUNCTION__))
5018 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5018, __extension__ __PRETTY_FUNCTION__));
5019 auto *Op = createOp(Builder, Name);
5020 switch (Kind) {
5021 case RK_Arithmetic:
5022 propagateIRFlags(Op, I);
5023 return Op;
5024 case RK_Min:
5025 case RK_Max:
5026 case RK_UMin:
5027 case RK_UMax:
5028 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5029 propagateIRFlags(SI->getCondition(),
5030 cast<SelectInst>(I)->getCondition());
5031 }
5032 propagateIRFlags(Op, I);
5033 return Op;
5034 case RK_None:
5035 break;
5036 }
5037 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5037);
5038 }
5039
5040 TargetTransformInfo::ReductionFlags getFlags() const {
5041 TargetTransformInfo::ReductionFlags Flags;
5042 Flags.NoNaN = NoNaN;
5043 switch (Kind) {
5044 case RK_Arithmetic:
5045 break;
5046 case RK_Min:
5047 Flags.IsSigned = Opcode == Instruction::ICmp;
5048 Flags.IsMaxOp = false;
5049 break;
5050 case RK_Max:
5051 Flags.IsSigned = Opcode == Instruction::ICmp;
5052 Flags.IsMaxOp = true;
5053 break;
5054 case RK_UMin:
5055 Flags.IsSigned = false;
5056 Flags.IsMaxOp = false;
5057 break;
5058 case RK_UMax:
5059 Flags.IsSigned = false;
5060 Flags.IsMaxOp = true;
5061 break;
5062 case RK_None:
5063 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5063);
5064 }
5065 return Flags;
5066 }
5067 };
5068
5069 Instruction *ReductionRoot = nullptr;
5070
5071 /// The operation data of the reduction operation.
5072 OperationData ReductionData;
5073
5074 /// The operation data of the values we perform a reduction on.
5075 OperationData ReducedValueData;
5076
5077 /// Should we model this reduction as a pairwise reduction tree or a tree that
5078 /// splits the vector in halves and adds those halves.
5079 bool IsPairwiseReduction = false;
5080
5081 /// Checks if the ParentStackElem.first should be marked as a reduction
5082 /// operation with an extra argument or as extra argument itself.
5083 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5084 Value *ExtraArg) {
5085 if (ExtraArgs.count(ParentStackElem.first)) {
5086 ExtraArgs[ParentStackElem.first] = nullptr;
5087 // We ran into something like:
5088 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5089 // The whole ParentStackElem.first should be considered as an extra value
5090 // in this case.
5091 // Do not perform analysis of remaining operands of ParentStackElem.first
5092 // instruction, this whole instruction is an extra argument.
5093 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5094 } else {
5095 // We ran into something like:
5096 // ParentStackElem.first += ... + ExtraArg + ...
5097 ExtraArgs[ParentStackElem.first] = ExtraArg;
5098 }
5099 }
5100
5101 static OperationData getOperationData(Value *V) {
5102 if (!V)
5103 return OperationData();
5104
5105 Value *LHS;
5106 Value *RHS;
5107 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5108 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5109 RK_Arithmetic);
5110 }
5111 if (auto *Select = dyn_cast<SelectInst>(V)) {
5112 // Look for a min/max pattern.
5113 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5114 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5115 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5116 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5117 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5118 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5119 return OperationData(
5120 Instruction::FCmp, LHS, RHS, RK_Min,
5121 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5122 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5123 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5124 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5125 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5126 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5127 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5128 return OperationData(
5129 Instruction::FCmp, LHS, RHS, RK_Max,
5130 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5131 }
5132 }
5133 return OperationData(V);
5134 }
5135
5136public:
5137 HorizontalReduction() = default;
5138
5139 /// \brief Try to find a reduction tree.
5140 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5141 assert((!Phi || is_contained(Phi->operands(), B)) &&(static_cast <bool> ((!Phi || is_contained(Phi->operands
(), B)) && "Thi phi needs to use the binary operator"
) ? void (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5142, __extension__ __PRETTY_FUNCTION__))
5142 "Thi phi needs to use the binary operator")(static_cast <bool> ((!Phi || is_contained(Phi->operands
(), B)) && "Thi phi needs to use the binary operator"
) ? void (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5142, __extension__ __PRETTY_FUNCTION__));
5143
5144 ReductionData = getOperationData(B);
5145
5146 // We could have a initial reductions that is not an add.
5147 // r *= v1 + v2 + v3 + v4
5148 // In such a case start looking for a tree rooted in the first '+'.
5149 if (Phi) {
5150 if (ReductionData.getLHS() == Phi) {
5151 Phi = nullptr;
5152 B = dyn_cast<Instruction>(ReductionData.getRHS());
5153 ReductionData = getOperationData(B);
5154 } else if (ReductionData.getRHS() == Phi) {
5155 Phi = nullptr;
5156 B = dyn_cast<Instruction>(ReductionData.getLHS());
5157 ReductionData = getOperationData(B);
5158 }
5159 }
5160
5161 if (!ReductionData.isVectorizable(B))
5162 return false;
5163
5164 Type *Ty = B->getType();
5165 if (!isValidElementType(Ty))
5166 return false;
5167
5168 ReducedValueData.clear();
5169 ReductionRoot = B;
5170
5171 // Post order traverse the reduction tree starting at B. We only handle true
5172 // trees containing only binary operators.
5173 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5174 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5175 ReductionData.initReductionOps(ReductionOps);
5176 while (!Stack.empty()) {
5177 Instruction *TreeN = Stack.back().first;
5178 unsigned EdgeToVist = Stack.back().second++;
5179 OperationData OpData = getOperationData(TreeN);
5180 bool IsReducedValue = OpData != ReductionData;
5181
5182 // Postorder vist.
5183 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5184 if (IsReducedValue)
5185 ReducedVals.push_back(TreeN);
5186 else {
5187 auto I = ExtraArgs.find(TreeN);
5188 if (I != ExtraArgs.end() && !I->second) {
5189 // Check if TreeN is an extra argument of its parent operation.
5190 if (Stack.size() <= 1) {
5191 // TreeN can't be an extra argument as it is a root reduction
5192 // operation.
5193 return false;
5194 }
5195 // Yes, TreeN is an extra argument, do not add it to a list of
5196 // reduction operations.
5197 // Stack[Stack.size() - 2] always points to the parent operation.
5198 markExtraArg(Stack[Stack.size() - 2], TreeN);
5199 ExtraArgs.erase(TreeN);
5200 } else
5201 ReductionData.addReductionOps(TreeN, ReductionOps);
5202 }
5203 // Retract.
5204 Stack.pop_back();
5205 continue;
5206 }
5207
5208 // Visit left or right.
5209 Value *NextV = TreeN->getOperand(EdgeToVist);
5210 if (NextV != Phi) {
5211 auto *I = dyn_cast<Instruction>(NextV);
5212 OpData = getOperationData(I);
5213 // Continue analysis if the next operand is a reduction operation or
5214 // (possibly) a reduced value. If the reduced value opcode is not set,
5215 // the first met operation != reduction operation is considered as the
5216 // reduced value class.
5217 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5218 OpData == ReductionData)) {
5219 const bool IsReductionOperation = OpData == ReductionData;
5220 // Only handle trees in the current basic block.
5221 if (!ReductionData.hasSameParent(I, B->getParent(),
5222 IsReductionOperation)) {
5223 // I is an extra argument for TreeN (its parent operation).
5224 markExtraArg(Stack.back(), I);
5225 continue;
5226 }
5227
5228 // Each tree node needs to have minimal number of users except for the
5229 // ultimate reduction.
5230 if (!ReductionData.hasRequiredNumberOfUses(I,
5231 OpData == ReductionData) &&
5232 I != B) {
5233 // I is an extra argument for TreeN (its parent operation).
5234 markExtraArg(Stack.back(), I);
5235 continue;
5236 }
5237
5238 if (IsReductionOperation) {
5239 // We need to be able to reassociate the reduction operations.
5240 if (!OpData.isAssociative(I)) {
5241 // I is an extra argument for TreeN (its parent operation).
5242 markExtraArg(Stack.back(), I);
5243 continue;
5244 }
5245 } else if (ReducedValueData &&
5246 ReducedValueData != OpData) {
5247 // Make sure that the opcodes of the operations that we are going to
5248 // reduce match.
5249 // I is an extra argument for TreeN (its parent operation).
5250 markExtraArg(Stack.back(), I);
5251 continue;
5252 } else if (!ReducedValueData)
5253 ReducedValueData = OpData;
5254
5255 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5256 continue;
5257 }
5258 }
5259 // NextV is an extra argument for TreeN (its parent operation).
5260 markExtraArg(Stack.back(), NextV);
5261 }
5262 return true;
5263 }
5264
5265 /// \brief Attempt to vectorize the tree found by
5266 /// matchAssociativeReduction.
5267 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5268 if (ReducedVals.empty())
5269 return false;
5270
5271 // If there is a sufficient number of reduction values, reduce
5272 // to a nearby power-of-2. Can safely generate oversized
5273 // vectors and rely on the backend to split them to legal sizes.
5274 unsigned NumReducedVals = ReducedVals.size();
5275 if (NumReducedVals < 4)
5276 return false;
5277
5278 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5279
5280 Value *VectorizedTree = nullptr;
5281 IRBuilder<> Builder(ReductionRoot);
5282 FastMathFlags Unsafe;
5283 Unsafe.setFast();
5284 Builder.setFastMathFlags(Unsafe);
5285 unsigned i = 0;
5286
5287 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5288 // The same extra argument may be used several time, so log each attempt
5289 // to use it.
5290 for (auto &Pair : ExtraArgs)
5291 ExternallyUsedValues[Pair.second].push_back(Pair.first);
5292 SmallVector<Value *, 16> IgnoreList;
5293 for (auto &V : ReductionOps)
5294 IgnoreList.append(V.begin(), V.end());
5295 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5296 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5297 V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5298 if (V.shouldReorder()) {
5299 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
5300 V.buildTree(Reversed, ExternallyUsedValues, IgnoreList);
5301 }
5302 if (V.isTreeTinyAndNotFullyVectorizable())
5303 break;
5304
5305 V.computeMinimumValueSizes();
5306
5307 // Estimate cost.
5308 int Cost =
5309 V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5310 if (Cost >= -SLPCostThreshold) {
5311 V.getORE()->emit([&]() {
5312 return OptimizationRemarkMissed(
5313 SV_NAME"slp-vectorizer", "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5314 << "Vectorizing horizontal reduction is possible"
5315 << "but not beneficial with cost "
5316 << ore::NV("Cost", Cost) << " and threshold "
5317 << ore::NV("Threshold", -SLPCostThreshold);
5318 });
5319 break;
5320 }
5321
5322 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
5323 << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false);
5324 V.getORE()->emit([&]() {
5325 return OptimizationRemark(
5326 SV_NAME"slp-vectorizer", "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5327 << "Vectorized horizontal reduction with cost "
5328 << ore::NV("Cost", Cost) << " and with tree size "
5329 << ore::NV("TreeSize", V.getTreeSize());
5330 });
5331
5332 // Vectorize a tree.
5333 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5334 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5335
5336 // Emit a reduction.
5337 Value *ReducedSubTree =
5338 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5339 if (VectorizedTree) {
5340 Builder.SetCurrentDebugLocation(Loc);
5341 OperationData VectReductionData(ReductionData.getOpcode(),
5342 VectorizedTree, ReducedSubTree,
5343 ReductionData.getKind());
5344 VectorizedTree =
5345 VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5346 } else
5347 VectorizedTree = ReducedSubTree;
5348 i += ReduxWidth;
5349 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5350 }
5351
5352 if (VectorizedTree) {
5353 // Finish the reduction.
5354 for (; i < NumReducedVals; ++i) {
5355 auto *I = cast<Instruction>(ReducedVals[i]);
5356 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5357 OperationData VectReductionData(ReductionData.getOpcode(),
5358 VectorizedTree, I,
5359 ReductionData.getKind());
5360 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5361 }
5362 for (auto &Pair : ExternallyUsedValues) {
5363 assert(!Pair.second.empty() &&(static_cast <bool> (!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? void (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5364, __extension__ __PRETTY_FUNCTION__))
5364 "At least one DebugLoc must be inserted")(static_cast <bool> (!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? void (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5364, __extension__ __PRETTY_FUNCTION__));
5365 // Add each externally used value to the final reduction.
5366 for (auto *I : Pair.second) {
5367 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5368 OperationData VectReductionData(ReductionData.getOpcode(),
5369 VectorizedTree, Pair.first,
5370 ReductionData.getKind());
5371 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5372 }
5373 }
5374 // Update users.
5375 ReductionRoot->replaceAllUsesWith(VectorizedTree);
5376 }
5377 return VectorizedTree != nullptr;
5378 }
5379
5380 unsigned numReductionValues() const {
5381 return ReducedVals.size();
5382 }
5383
5384private:
5385 /// \brief Calculate the cost of a reduction.
5386 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5387 unsigned ReduxWidth) {
5388 Type *ScalarTy = FirstReducedVal->getType();
5389 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5390
5391 int PairwiseRdxCost;
5392 int SplittingRdxCost;
5393 switch (ReductionData.getKind()) {
5394 case RK_Arithmetic:
5395 PairwiseRdxCost =
5396 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5397 /*IsPairwiseForm=*/true);
5398 SplittingRdxCost =
5399 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5400 /*IsPairwiseForm=*/false);
5401 break;
5402 case RK_Min:
5403 case RK_Max:
5404 case RK_UMin:
5405 case RK_UMax: {
5406 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5407 bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5408 ReductionData.getKind() == RK_UMax;
5409 PairwiseRdxCost =
5410 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5411 /*IsPairwiseForm=*/true, IsUnsigned);
5412 SplittingRdxCost =
5413 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5414 /*IsPairwiseForm=*/false, IsUnsigned);
5415 break;
5416 }
5417 case RK_None:
5418 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5418);
5419 }
5420
5421 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5422 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5423
5424 int ScalarReduxCost;
5425 switch (ReductionData.getKind()) {
5426 case RK_Arithmetic:
5427 ScalarReduxCost =
5428 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5429 break;
5430 case RK_Min:
5431 case RK_Max:
5432 case RK_UMin:
5433 case RK_UMax:
5434 ScalarReduxCost =
5435 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5436 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5437 CmpInst::makeCmpResultType(ScalarTy));
5438 break;
5439 case RK_None:
5440 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5440);
5441 }
5442 ScalarReduxCost *= (ReduxWidth - 1);
5443
5444 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5445 << " for reduction that starts with " << *FirstReducedValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5446 << " (It is a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5447 << (IsPairwiseReduction ? "pairwise" : "splitting")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5448 << " reduction)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false);
5449
5450 return VecReduxCost - ScalarReduxCost;
5451 }
5452
5453 /// \brief Emit a horizontal reduction of the vectorized value.
5454 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5455 unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5456 assert(VectorizedValue && "Need to have a vectorized tree node")(static_cast <bool> (VectorizedValue && "Need to have a vectorized tree node"
) ? void (0) : __assert_fail ("VectorizedValue && \"Need to have a vectorized tree node\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5456, __extension__ __PRETTY_FUNCTION__));
5457 assert(isPowerOf2_32(ReduxWidth) &&(static_cast <bool> (isPowerOf2_32(ReduxWidth) &&
"We only handle power-of-two reductions for now") ? void (0)
: __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5458, __extension__ __PRETTY_FUNCTION__))
5458 "We only handle power-of-two reductions for now")(static_cast <bool> (isPowerOf2_32(ReduxWidth) &&
"We only handle power-of-two reductions for now") ? void (0)
: __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5458, __extension__ __PRETTY_FUNCTION__));
5459
5460 if (!IsPairwiseReduction)
5461 return createSimpleTargetReduction(
5462 Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5463 ReductionData.getFlags(), ReductionOps.back());
5464
5465 Value *TmpVec = VectorizedValue;
5466 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5467 Value *LeftMask =
5468 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5469 Value *RightMask =
5470 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5471
5472 Value *LeftShuf = Builder.CreateShuffleVector(
5473 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5474 Value *RightShuf = Builder.CreateShuffleVector(
5475 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5476 "rdx.shuf.r");
5477 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5478 RightShuf, ReductionData.getKind());
5479 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5480 }
5481
5482 // The result is in the first element of the vector.
5483 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5484 }
5485};
5486
5487} // end anonymous namespace
5488
5489/// \brief Recognize construction of vectors like
5490/// %ra = insertelement <4 x float> undef, float %s0, i32 0
5491/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5492/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5493/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5494/// starting from the last insertelement instruction.
5495///
5496/// Returns true if it matches
5497static bool findBuildVector(InsertElementInst *LastInsertElem,
5498 SmallVectorImpl<Value *> &BuildVector,
5499 SmallVectorImpl<Value *> &BuildVectorOpds) {
5500 Value *V = nullptr;
5501 do {
5502 BuildVector.push_back(LastInsertElem);
5503 BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5504 V = LastInsertElem->getOperand(0);
5505 if (isa<UndefValue>(V))
5506 break;
5507 LastInsertElem = dyn_cast<InsertElementInst>(V);
5508 if (!LastInsertElem || !LastInsertElem->hasOneUse())
5509 return false;
5510 } while (true);
5511 std::reverse(BuildVector.begin(), BuildVector.end());
5512 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5513 return true;
5514}
5515
5516/// \brief Like findBuildVector, but looks for construction of aggregate.
5517///
5518/// \return true if it matches.
5519static bool findBuildAggregate(InsertValueInst *IV,
5520 SmallVectorImpl<Value *> &BuildVector,
5521 SmallVectorImpl<Value *> &BuildVectorOpds) {
5522 Value *V;
5523 do {
5524 BuildVector.push_back(IV);
5525 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
5526 V = IV->getAggregateOperand();
5527 if (isa<UndefValue>(V))
5528 break;
5529 IV = dyn_cast<InsertValueInst>(V);
5530 if (!IV || !IV->hasOneUse())
5531 return false;
5532 } while (true);
5533 std::reverse(BuildVector.begin(), BuildVector.end());
5534 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5535 return true;
5536}
5537
5538static bool PhiTypeSorterFunc(Value *V, Value *V2) {
5539 return V->getType() < V2->getType();
5540}
5541
5542/// \brief Try and get a reduction value from a phi node.
5543///
5544/// Given a phi node \p P in a block \p ParentBB, consider possible reductions
5545/// if they come from either \p ParentBB or a containing loop latch.
5546///
5547/// \returns A candidate reduction value if possible, or \code nullptr \endcode
5548/// if not possible.
5549static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
5550 BasicBlock *ParentBB, LoopInfo *LI) {
5551 // There are situations where the reduction value is not dominated by the
5552 // reduction phi. Vectorizing such cases has been reported to cause
5553 // miscompiles. See PR25787.
5554 auto DominatedReduxValue = [&](Value *R) {
5555 return (
5556 dyn_cast<Instruction>(R) &&
5557 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
5558 };
5559
5560 Value *Rdx = nullptr;
5561
5562 // Return the incoming value if it comes from the same BB as the phi node.
5563 if (P->getIncomingBlock(0) == ParentBB) {
5564 Rdx = P->getIncomingValue(0);
5565 } else if (P->getIncomingBlock(1) == ParentBB) {
5566 Rdx = P->getIncomingValue(1);
5567 }
5568
5569 if (Rdx && DominatedReduxValue(Rdx))
5570 return Rdx;
5571
5572 // Otherwise, check whether we have a loop latch to look at.
5573 Loop *BBL = LI->getLoopFor(ParentBB);
5574 if (!BBL)
5575 return nullptr;
5576 BasicBlock *BBLatch = BBL->getLoopLatch();
5577 if (!BBLatch)
5578 return nullptr;
5579
5580 // There is a loop latch, return the incoming value if it comes from
5581 // that. This reduction pattern occasionally turns up.
5582 if (P->getIncomingBlock(0) == BBLatch) {
5583 Rdx = P->getIncomingValue(0);
5584 } else if (P->getIncomingBlock(1) == BBLatch) {
5585 Rdx = P->getIncomingValue(1);
5586 }
5587
5588 if (Rdx && DominatedReduxValue(Rdx))
5589 return Rdx;
5590
5591 return nullptr;
5592}
5593
5594/// Attempt to reduce a horizontal reduction.
5595/// If it is legal to match a horizontal reduction feeding the phi node \a P
5596/// with reduction operators \a Root (or one of its operands) in a basic block
5597/// \a BB, then check if it can be done. If horizontal reduction is not found
5598/// and root instruction is a binary operation, vectorization of the operands is
5599/// attempted.
5600/// \returns true if a horizontal reduction was matched and reduced or operands
5601/// of one of the binary instruction were vectorized.
5602/// \returns false if a horizontal reduction was not matched (or not possible)
5603/// or no vectorization of any binary operation feeding \a Root instruction was
5604/// performed.
5605static bool tryToVectorizeHorReductionOrInstOperands(
5606 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
5607 TargetTransformInfo *TTI,
5608 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
5609 if (!ShouldVectorizeHor)
5610 return false;
5611
5612 if (!Root)
5613 return false;
5614
5615 if (Root->getParent() != BB || isa<PHINode>(Root))
5616 return false;
5617 // Start analysis starting from Root instruction. If horizontal reduction is
5618 // found, try to vectorize it. If it is not a horizontal reduction or
5619 // vectorization is not possible or not effective, and currently analyzed
5620 // instruction is a binary operation, try to vectorize the operands, using
5621 // pre-order DFS traversal order. If the operands were not vectorized, repeat
5622 // the same procedure considering each operand as a possible root of the
5623 // horizontal reduction.
5624 // Interrupt the process if the Root instruction itself was vectorized or all
5625 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
5626 SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
5627 SmallSet<Value *, 8> VisitedInstrs;
5628 bool Res = false;
5629 while (!Stack.empty()) {
5630 Value *V;
5631 unsigned Level;
5632 std::tie(V, Level) = Stack.pop_back_val();
5633 if (!V)
5634 continue;
5635 auto *Inst = dyn_cast<Instruction>(V);
5636 if (!Inst)
5637 continue;
5638 auto *BI = dyn_cast<BinaryOperator>(Inst);
5639 auto *SI = dyn_cast<SelectInst>(Inst);
5640 if (BI || SI) {
5641 HorizontalReduction HorRdx;
5642 if (HorRdx.matchAssociativeReduction(P, Inst)) {
5643 if (HorRdx.tryToReduce(R, TTI)) {
5644 Res = true;
5645 // Set P to nullptr to avoid re-analysis of phi node in
5646 // matchAssociativeReduction function unless this is the root node.
5647 P = nullptr;
5648 continue;
5649 }
5650 }
5651 if (P && BI) {
5652 Inst = dyn_cast<Instruction>(BI->getOperand(0));
5653 if (Inst == P)
5654 Inst = dyn_cast<Instruction>(BI->getOperand(1));
5655 if (!Inst) {
5656 // Set P to nullptr to avoid re-analysis of phi node in
5657 // matchAssociativeReduction function unless this is the root node.
5658 P = nullptr;
5659 continue;
5660 }
5661 }
5662 }
5663 // Set P to nullptr to avoid re-analysis of phi node in
5664 // matchAssociativeReduction function unless this is the root node.
5665 P = nullptr;
5666 if (Vectorize(Inst, R)) {
5667 Res = true;
5668 continue;
5669 }
5670
5671 // Try to vectorize operands.
5672 // Continue analysis for the instruction from the same basic block only to
5673 // save compile time.
5674 if (++Level < RecursionMaxDepth)
5675 for (auto *Op : Inst->operand_values())
5676 if (VisitedInstrs.insert(Op).second)
5677 if (auto *I = dyn_cast<Instruction>(Op))
5678 if (!isa<PHINode>(I) && I->getParent() == BB)
5679 Stack.emplace_back(Op, Level);
5680 }
5681 return Res;
5682}
5683
5684bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
5685 BasicBlock *BB, BoUpSLP &R,
5686 TargetTransformInfo *TTI) {
5687 if (!V)
5688 return false;
5689 auto *I = dyn_cast<Instruction>(V);
5690 if (!I)
5691 return false;
5692
5693 if (!isa<BinaryOperator>(I))
5694 P = nullptr;
5695 // Try to match and vectorize a horizontal reduction.
5696 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
5697 return tryToVectorize(I, R);
5698 };
5699 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
5700 ExtraVectorization);
5701}
5702
5703bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
5704 BasicBlock *BB, BoUpSLP &R) {
5705 const DataLayout &DL = BB->getModule()->getDataLayout();
5706 if (!R.canMapToVector(IVI->getType(), DL))
5707 return false;
5708
5709 SmallVector<Value *, 16> BuildVector;
5710 SmallVector<Value *, 16> BuildVectorOpds;
5711 if (!findBuildAggregate(IVI, BuildVector, BuildVectorOpds))
5712 return false;
5713
5714 DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: array mappable to vector: " <<
*IVI << "\n"; } } while (false);
5715 // Aggregate value is unlikely to be processed in vector register, we need to
5716 // extract scalars into scalar registers, so NeedExtraction is set true.
5717 return tryToVectorizeList(BuildVectorOpds, R, BuildVector, false, true);
5718}
5719
5720bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
5721 BasicBlock *BB, BoUpSLP &R) {
5722 SmallVector<Value *, 16> BuildVector;
5723 SmallVector<Value *, 16> BuildVectorOpds;
5724 if (!findBuildVector(IEI, BuildVector, BuildVectorOpds))
5725 return false;
5726
5727 // Vectorize starting with the build vector operands ignoring the BuildVector
5728 // instructions for the purpose of scheduling and user extraction.
5729 return tryToVectorizeList(BuildVectorOpds, R, BuildVector);
5730}
5731
5732bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
5733 BoUpSLP &R) {
5734 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
5735 return true;
5736
5737 bool OpsChanged = false;
5738 for (int Idx = 0; Idx < 2; ++Idx) {
5739 OpsChanged |=
5740 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
5741 }
5742 return OpsChanged;
5743}
5744
5745bool SLPVectorizerPass::vectorizeSimpleInstructions(
5746 SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
5747 bool OpsChanged = false;
5748 for (auto &VH : reverse(Instructions)) {
5749 auto *I = dyn_cast_or_null<Instruction>(VH);
5750 if (!I)
5751 continue;
5752 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
5753 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
5754 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
5755 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
5756 else if (auto *CI = dyn_cast<CmpInst>(I))
5757 OpsChanged |= vectorizeCmpInst(CI, BB, R);
5758 }
5759 Instructions.clear();
5760 return OpsChanged;
5761}
5762
5763bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
5764 bool Changed = false;
5765 SmallVector<Value *, 4> Incoming;
5766 SmallSet<Value *, 16> VisitedInstrs;
5767
5768 bool HaveVectorizedPhiNodes = true;
5769 while (HaveVectorizedPhiNodes) {
5770 HaveVectorizedPhiNodes = false;
5771
5772 // Collect the incoming values from the PHIs.
5773 Incoming.clear();
5774 for (Instruction &I : *BB) {
5775 PHINode *P = dyn_cast<PHINode>(&I);
5776 if (!P)
5777 break;
5778
5779 if (!VisitedInstrs.count(P))
5780 Incoming.push_back(P);
5781 }
5782
5783 // Sort by type.
5784 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
5785
5786 // Try to vectorize elements base on their type.
5787 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
5788 E = Incoming.end();
5789 IncIt != E;) {
5790
5791 // Look for the next elements with the same type.
5792 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
5793 while (SameTypeIt != E &&
5794 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
5795 VisitedInstrs.insert(*SameTypeIt);
5796 ++SameTypeIt;
5797 }
5798
5799 // Try to vectorize them.
5800 unsigned NumElts = (SameTypeIt - IncIt);
5801 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { errs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false);
5802 // The order in which the phi nodes appear in the program does not matter.
5803 // So allow tryToVectorizeList to reorder them if it is beneficial. This
5804 // is done when there are exactly two elements since tryToVectorizeList
5805 // asserts that there are only two values when AllowReorder is true.
5806 bool AllowReorder = NumElts == 2;
5807 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
5808 None, AllowReorder)) {
5809 // Success start over because instructions might have been changed.
5810 HaveVectorizedPhiNodes = true;
5811 Changed = true;
5812 break;
5813 }
5814
5815 // Start over at the next instruction of a different type (or the end).
5816 IncIt = SameTypeIt;
5817 }
5818 }
5819
5820 VisitedInstrs.clear();
5821
5822 SmallVector<WeakVH, 8> PostProcessInstructions;
5823 SmallDenseSet<Instruction *, 4> KeyNodes;
5824 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
5825 // We may go through BB multiple times so skip the one we have checked.
5826 if (!VisitedInstrs.insert(&*it).second) {
5827 if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
5828 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
5829 // We would like to start over since some instructions are deleted
5830 // and the iterator may become invalid value.
5831 Changed = true;
5832 it = BB->begin();
5833 e = BB->end();
5834 }
5835 continue;
5836 }
5837
5838 if (isa<DbgInfoIntrinsic>(it))
5839 continue;
5840
5841 // Try to vectorize reductions that use PHINodes.
5842 if (PHINode *P = dyn_cast<PHINode>(it)) {
5843 // Check that the PHI is a reduction PHI.
5844 if (P->getNumIncomingValues() != 2)
5845 return Changed;
5846
5847 // Try to match and vectorize a horizontal reduction.
5848 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
5849 TTI)) {
5850 Changed = true;
5851 it = BB->begin();
5852 e = BB->end();
5853 continue;
5854 }
5855 continue;
5856 }
5857
5858 // Ran into an instruction without users, like terminator, or function call
5859 // with ignored return value, store. Ignore unused instructions (basing on
5860 // instruction type, except for CallInst and InvokeInst).
5861 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
5862 isa<InvokeInst>(it))) {
5863 KeyNodes.insert(&*it);
5864 bool OpsChanged = false;
5865 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
5866 for (auto *V : it->operand_values()) {
5867 // Try to match and vectorize a horizontal reduction.
5868 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
5869 }
5870 }
5871 // Start vectorization of post-process list of instructions from the
5872 // top-tree instructions to try to vectorize as many instructions as
5873 // possible.
5874 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
5875 if (OpsChanged) {
5876 // We would like to start over since some instructions are deleted
5877 // and the iterator may become invalid value.
5878 Changed = true;
5879 it = BB->begin();
5880 e = BB->end();
5881 continue;
5882 }
5883 }
5884
5885 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
5886 isa<InsertValueInst>(it))
5887 PostProcessInstructions.push_back(&*it);
5888
5889 }
5890
5891 return Changed;
5892}
5893
5894bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
5895 auto Changed = false;
5896 for (auto &Entry : GEPs) {
5897 // If the getelementptr list has fewer than two elements, there's nothing
5898 // to do.
5899 if (Entry.second.size() < 2)
5900 continue;
5901
5902 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
5903 << Entry.second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
);
5904
5905 // We process the getelementptr list in chunks of 16 (like we do for
5906 // stores) to minimize compile-time.
5907 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
5908 auto Len = std::min<unsigned>(BE - BI, 16);
5909 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
5910
5911 // Initialize a set a candidate getelementptrs. Note that we use a
5912 // SetVector here to preserve program order. If the index computations
5913 // are vectorizable and begin with loads, we want to minimize the chance
5914 // of having to reorder them later.
5915 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
5916
5917 // Some of the candidates may have already been vectorized after we
5918 // initially collected them. If so, the WeakTrackingVHs will have
5919 // nullified the
5920 // values, so remove them from the set of candidates.
5921 Candidates.remove(nullptr);
5922
5923 // Remove from the set of candidates all pairs of getelementptrs with
5924 // constant differences. Such getelementptrs are likely not good
5925 // candidates for vectorization in a bottom-up phase since one can be
5926 // computed from the other. We also ensure all candidate getelementptr
5927 // indices are unique.
5928 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
5929 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
5930 if (!Candidates.count(GEPI))
5931 continue;
5932 auto *SCEVI = SE->getSCEV(GEPList[I]);
5933 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
5934 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
5935 auto *SCEVJ = SE->getSCEV(GEPList[J]);
5936 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
5937 Candidates.remove(GEPList[I]);
5938 Candidates.remove(GEPList[J]);
5939 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
5940 Candidates.remove(GEPList[J]);
5941 }
5942 }
5943 }
5944
5945 // We break out of the above computation as soon as we know there are
5946 // fewer than two candidates remaining.
5947 if (Candidates.size() < 2)
5948 continue;
5949
5950 // Add the single, non-constant index of each candidate to the bundle. We
5951 // ensured the indices met these constraints when we originally collected
5952 // the getelementptrs.
5953 SmallVector<Value *, 16> Bundle(Candidates.size());
5954 auto BundleIndex = 0u;
5955 for (auto *V : Candidates) {
5956 auto *GEP = cast<GetElementPtrInst>(V);
5957 auto *GEPIdx = GEP->idx_begin()->get();
5958 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx))(static_cast <bool> (GEP->getNumIndices() == 1 || !isa
<Constant>(GEPIdx)) ? void (0) : __assert_fail ("GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5958, __extension__ __PRETTY_FUNCTION__));
5959 Bundle[BundleIndex++] = GEPIdx;
5960 }
5961
5962 // Try and vectorize the indices. We are currently only interested in
5963 // gather-like cases of the form:
5964 //
5965 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
5966 //
5967 // where the loads of "a", the loads of "b", and the subtractions can be
5968 // performed in parallel. It's likely that detecting this pattern in a
5969 // bottom-up phase will be simpler and less costly than building a
5970 // full-blown top-down phase beginning at the consecutive loads.
5971 Changed |= tryToVectorizeList(Bundle, R);
5972 }
5973 }
5974 return Changed;
5975}
5976
5977bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
5978 bool Changed = false;
5979 // Attempt to sort and vectorize each of the store-groups.
5980 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
5981 ++it) {
5982 if (it->second.size() < 2)
5983 continue;
5984
5985 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
5986 << it->second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
);
5987
5988 // Process the stores in chunks of 16.
5989 // TODO: The limit of 16 inhibits greater vectorization factors.
5990 // For example, AVX2 supports v32i8. Increasing this limit, however,
5991 // may cause a significant compile-time increase.
5992 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
5993 unsigned Len = std::min<unsigned>(CE - CI, 16);
5994 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
5995 }
5996 }
5997 return Changed;
5998}
5999
6000char SLPVectorizer::ID = 0;
6001
6002static const char lv_name[] = "SLP Vectorizer";
6003
6004INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)static void *initializeSLPVectorizerPassOnce(PassRegistry &
Registry) {
6005INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
6006INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
6007INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
6008INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
6009INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry);
6010INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
6011INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
6012INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "slp-vectorizer", &
SLPVectorizer::ID, PassInfo::NormalCtor_t(callDefaultCtor<
SLPVectorizer>), false, false); Registry.registerPass(*PI,
true); return PI; } static llvm::once_flag InitializeSLPVectorizerPassFlag
; void llvm::initializeSLPVectorizerPass(PassRegistry &Registry
) { llvm::call_once(InitializeSLPVectorizerPassFlag, initializeSLPVectorizerPassOnce
, std::ref(Registry)); }
6013
6014Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); }
←

/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h

→
1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
11// and dyn_cast_or_null<X>() templates.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_SUPPORT_CASTING_H
16#define LLVM_SUPPORT_CASTING_H
17
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/type_traits.h"
20#include <cassert>
21#include <memory>
22#include <type_traits>
23
24namespace llvm {
25
26//===----------------------------------------------------------------------===//
27// isa<x> Support Templates
28//===----------------------------------------------------------------------===//
29
30// Define a template that can be specialized by smart pointers to reflect the
31// fact that they are automatically dereferenced, and are not involved with the
32// template selection process... the default implementation is a noop.
33//
34template<typename From> struct simplify_type {
35 using SimpleType = From; // The real type this represents...
36
37 // An accessor to get the real value...
38 static SimpleType &getSimplifiedValue(From &Val) { return Val; }
39};
40
41template<typename From> struct simplify_type<const From> {
42 using NonConstSimpleType = typename simplify_type<From>::SimpleType;
43 using SimpleType =
44 typename add_const_past_pointer<NonConstSimpleType>::type;
45 using RetType =
46 typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
47
48 static RetType getSimplifiedValue(const From& Val) {
49 return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
12
←
Calling 'simplify_type::getSimplifiedValue'
→
13
←
Returning from 'simplify_type::getSimplifiedValue'
→
50 }
51};
52
53// The core of the implementation of isa<X> is here; To and From should be
54// the names of classes. This template can be specialized to customize the
55// implementation of isa<> without rewriting it from scratch.
56template <typename To, typename From, typename Enabler = void>
57struct isa_impl {
58 static inline bool doit(const From &Val) {
59 return To::classof(&Val);
19
←
Calling 'CallInst::classof'
→
25
←
Returning from 'CallInst::classof'
→
60 }
61};
62
63/// \brief Always allow upcasts, and perform no dynamic check for them.
64template <typename To, typename From>
65struct isa_impl<
66 To, From, typename std::enable_if<std::is_base_of<To, From>::value>::type> {
67 static inline bool doit(const From &) { return true; }
68};
69
70template <typename To, typename From> struct isa_impl_cl {
71 static inline bool doit(const From &Val) {
72 return isa_impl<To, From>::doit(Val);
73 }
74};
75
76template <typename To, typename From> struct isa_impl_cl<To, const From> {
77 static inline bool doit(const From &Val) {
78 return isa_impl<To, From>::doit(Val);
79 }
80};
81
82template <typename To, typename From>
83struct isa_impl_cl<To, const std::unique_ptr<From>> {
84 static inline bool doit(const std::unique_ptr<From> &Val) {
85 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 85, __extension__ __PRETTY_FUNCTION__));
86 return isa_impl_cl<To, From>::doit(*Val);
87 }
88};
89
90template <typename To, typename From> struct isa_impl_cl<To, From*> {
91 static inline bool doit(const From *Val) {
92 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 92, __extension__ __PRETTY_FUNCTION__));
93 return isa_impl<To, From>::doit(*Val);
94 }
95};
96
97template <typename To, typename From> struct isa_impl_cl<To, From*const> {
98 static inline bool doit(const From *Val) {
99 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 99, __extension__ __PRETTY_FUNCTION__));
100 return isa_impl<To, From>::doit(*Val);
101 }
102};
103
104template <typename To, typename From> struct isa_impl_cl<To, const From*> {
105 static inline bool doit(const From *Val) {
106 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 106, __extension__ __PRETTY_FUNCTION__));
17
←
Within the expansion of the macro 'assert':
→
107 return isa_impl<To, From>::doit(*Val);
18
←
Calling 'isa_impl::doit'
→
26
←
Returning from 'isa_impl::doit'
→
108 }
109};
110
111template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
112 static inline bool doit(const From *Val) {
113 assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 113, __extension__ __PRETTY_FUNCTION__));
114 return isa_impl<To, From>::doit(*Val);
115 }
116};
117
118template<typename To, typename From, typename SimpleFrom>
119struct isa_impl_wrap {
120 // When From != SimplifiedType, we can simplify the type some more by using
121 // the simplify_type template.
122 static bool doit(const From &Val) {
123 return isa_impl_wrap<To, SimpleFrom,
15
←
Calling 'isa_impl_wrap::doit'
→
28
←
Returning from 'isa_impl_wrap::doit'
→
124 typename simplify_type<SimpleFrom>::SimpleType>::doit(
125 simplify_type<const From>::getSimplifiedValue(Val));
11
←
Calling 'simplify_type::getSimplifiedValue'
→
14
←
Returning from 'simplify_type::getSimplifiedValue'
→
126 }
127};
128
129template<typename To, typename FromTy>
130struct isa_impl_wrap<To, FromTy, FromTy> {
131 // When From == SimpleType, we are as simple as we are going to get.
132 static bool doit(const FromTy &Val) {
133 return isa_impl_cl<To,FromTy>::doit(Val);
16
←
Calling 'isa_impl_cl::doit'
→
27
←
Returning from 'isa_impl_cl::doit'
→
134 }
135};
136
137// isa<X> - Return true if the parameter to the template is an instance of the
138// template type argument. Used like this:
139//
140// if (isa<Type>(myVal)) { ... }
141//
142template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
143 return isa_impl_wrap<X, const Y,
10
←
Calling 'isa_impl_wrap::doit'
→
29
←
Returning from 'isa_impl_wrap::doit'
→
144 typename simplify_type<const Y>::SimpleType>::doit(Val);
145}
146
147//===----------------------------------------------------------------------===//
148// cast<x> Support Templates
149//===----------------------------------------------------------------------===//
150
151template<class To, class From> struct cast_retty;
152
153// Calculate what type the 'cast' function should return, based on a requested
154// type of To and a source type of From.
155template<class To, class From> struct cast_retty_impl {
156 using ret_type = To &; // Normal case, return Ty&
157};
158template<class To, class From> struct cast_retty_impl<To, const From> {
159 using ret_type = const To &; // Normal case, return Ty&
160};
161
162template<class To, class From> struct cast_retty_impl<To, From*> {
163 using ret_type = To *; // Pointer arg case, return Ty*
164};
165
166template<class To, class From> struct cast_retty_impl<To, const From*> {
167 using ret_type = const To *; // Constant pointer arg case, return const Ty*
168};
169
170template<class To, class From> struct cast_retty_impl<To, const From*const> {
171 using ret_type = const To *; // Constant pointer arg case, return const Ty*
172};
173
174template <class To, class From>
175struct cast_retty_impl<To, std::unique_ptr<From>> {
176private:
177 using PointerType = typename cast_retty_impl<To, From *>::ret_type;
178 using ResultType = typename std::remove_pointer<PointerType>::type;
179
180public:
181 using ret_type = std::unique_ptr<ResultType>;
182};
183
184template<class To, class From, class SimpleFrom>
185struct cast_retty_wrap {
186 // When the simplified type and the from type are not the same, use the type
187 // simplifier to reduce the type, then reuse cast_retty_impl to get the
188 // resultant type.
189 using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
190};
191
192template<class To, class FromTy>
193struct cast_retty_wrap<To, FromTy, FromTy> {
194 // When the simplified type is equal to the from type, use it directly.
195 using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
196};
197
198template<class To, class From>
199struct cast_retty {
200 using ret_type = typename cast_retty_wrap<
201 To, From, typename simplify_type<From>::SimpleType>::ret_type;
202};
203
204// Ensure the non-simple values are converted using the simplify_type template
205// that may be specialized by smart pointers...
206//
207template<class To, class From, class SimpleFrom> struct cast_convert_val {
208 // This is not a simple type, use the template to simplify it...
209 static typename cast_retty<To, From>::ret_type doit(From &Val) {
210 return cast_convert_val<To, SimpleFrom,
211 typename simplify_type<SimpleFrom>::SimpleType>::doit(
212 simplify_type<From>::getSimplifiedValue(Val));
213 }
214};
215
216template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
217 // This _is_ a simple type, just cast it.
218 static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
219 typename cast_retty<To, FromTy>::ret_type Res2
220 = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
221 return Res2;
222 }
223};
224
225template <class X> struct is_simple_type {
226 static const bool value =
227 std::is_same<X, typename simplify_type<X>::SimpleType>::value;
228};
229
230// cast<X> - Return the argument parameter cast to the specified type. This
231// casting operator asserts that the type is correct, so it does not return null
232// on failure. It does not allow a null argument (use cast_or_null for that).
233// It is typically used like this:
234//
235// cast<Instruction>(myVal)->getParent()
236//
237template <class X, class Y>
238inline typename std::enable_if<!is_simple_type<Y>::value,
239 typename cast_retty<X, const Y>::ret_type>::type
240cast(const Y &Val) {
241 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 241, __extension__ __PRETTY_FUNCTION__));
242 return cast_convert_val<
243 X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
244}
245
246template <class X, class Y>
247inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
248 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 248, __extension__ __PRETTY_FUNCTION__));
249 return cast_convert_val<X, Y,
250 typename simplify_type<Y>::SimpleType>::doit(Val);
251}
252
253template <class X, class Y>
254inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
255 assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 255, __extension__ __PRETTY_FUNCTION__));
9
←
Within the expansion of the macro 'assert':
→
a
Calling 'isa'
b
Returning from 'isa'
256 return cast_convert_val<X, Y*,
30
←
Calling 'cast_convert_val::doit'
→
31
←
Returning from 'cast_convert_val::doit'
→
257 typename simplify_type<Y*>::SimpleType>::doit(Val);
258}
259
260template <class X, class Y>
261inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
262cast(std::unique_ptr<Y> &&Val) {
263 assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val.get()) &&
"cast<Ty>() argument of incompatible type!") ? void (0
) : __assert_fail ("isa<X>(Val.get()) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 263, __extension__ __PRETTY_FUNCTION__));
264 using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
265 return ret_type(
266 cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
267 Val.release()));
268}
269
270// cast_or_null<X> - Functionally identical to cast, except that a null value is
271// accepted.
272//
273template <class X, class Y>
274LLVM_NODISCARD[[clang::warn_unused_result]] inline
275 typename std::enable_if<!is_simple_type<Y>::value,
276 typename cast_retty<X, const Y>::ret_type>::type
277 cast_or_null(const Y &Val) {
278 if (!Val)
279 return nullptr;
280 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 280, __extension__ __PRETTY_FUNCTION__));
281 return cast<X>(Val);
282}
283
284template <class X, class Y>
285LLVM_NODISCARD[[clang::warn_unused_result]] inline
286 typename std::enable_if<!is_simple_type<Y>::value,
287 typename cast_retty<X, Y>::ret_type>::type
288 cast_or_null(Y &Val) {
289 if (!Val)
290 return nullptr;
291 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 291, __extension__ __PRETTY_FUNCTION__));
292 return cast<X>(Val);
293}
294
295template <class X, class Y>
296LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
297cast_or_null(Y *Val) {
298 if (!Val) return nullptr;
299 assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/Support/Casting.h"
, 299, __extension__ __PRETTY_FUNCTION__));
300 return cast<X>(Val);
301}
302
303template <class X, class Y>
304inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
305cast_or_null(std::unique_ptr<Y> &&Val) {
306 if (!Val)
307 return nullptr;
308 return cast<X>(std::move(Val));
309}
310
311// dyn_cast<X> - Return the argument parameter cast to the specified type. This
312// casting operator returns null if the argument is of the wrong type, so it can
313// be used to test for a type as well as cast if successful. This should be
314// used in the context of an if statement like this:
315//
316// if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
317//
318
319template <class X, class Y>
320LLVM_NODISCARD[[clang::warn_unused_result]] inline
321 typename std::enable_if<!is_simple_type<Y>::value,
322 typename cast_retty<X, const Y>::ret_type>::type
323 dyn_cast(const Y &Val) {
324 return isa<X>(Val) ? cast<X>(Val) : nullptr;
325}
326
327template <class X, class Y>
328LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
329 return isa<X>(Val) ? cast<X>(Val) : nullptr;
330}
331
332template <class X, class Y>
333LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
334 return isa<X>(Val) ? cast<X>(Val) : nullptr;
335}
336
337// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
338// value is accepted.
339//
340template <class X, class Y>
341LLVM_NODISCARD[[clang::warn_unused_result]] inline
342 typename std::enable_if<!is_simple_type<Y>::value,
343 typename cast_retty<X, const Y>::ret_type>::type
344 dyn_cast_or_null(const Y &Val) {
345 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
346}
347
348template <class X, class Y>
349LLVM_NODISCARD[[clang::warn_unused_result]] inline
350 typename std::enable_if<!is_simple_type<Y>::value,
351 typename cast_retty<X, Y>::ret_type>::type
352 dyn_cast_or_null(Y &Val) {
353 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
354}
355
356template <class X, class Y>
357LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
358dyn_cast_or_null(Y *Val) {
359 return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
360}
361
362// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
363// taking ownership of the input pointer iff isa<X>(Val) is true. If the
364// cast is successful, From refers to nullptr on exit and the casted value
365// is returned. If the cast is unsuccessful, the function returns nullptr
366// and From is unchanged.
367template <class X, class Y>
368LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
369 -> decltype(cast<X>(Val)) {
370 if (!isa<X>(Val))
371 return nullptr;
372 return cast<X>(std::move(Val));
373}
374
375template <class X, class Y>
376LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val)
377 -> decltype(cast<X>(Val)) {
378 return unique_dyn_cast<X, Y>(Val);
379}
380
381// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
382// a null value is accepted.
383template <class X, class Y>
384LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
385 -> decltype(cast<X>(Val)) {
386 if (!Val)
387 return nullptr;
388 return unique_dyn_cast<X, Y>(Val);
389}
390
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val)
393 -> decltype(cast<X>(Val)) {
394 return unique_dyn_cast_or_null<X, Y>(Val);
395}
396
397} // end namespace llvm
398
399#endif // LLVM_SUPPORT_CASTING_H
←

/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h

→
1//===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file exposes the class definitions of all of the subclasses of the
11// Instruction class. This is meant to be an easy way to get access to all
12// instruction subclasses.
13//
14//===----------------------------------------------------------------------===//
15
16#ifndef LLVM_IR_INSTRUCTIONS_H
17#define LLVM_IR_INSTRUCTIONS_H
18
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/None.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/ADT/iterator.h"
26#include "llvm/ADT/iterator_range.h"
27#include "llvm/IR/Attributes.h"
28#include "llvm/IR/BasicBlock.h"
29#include "llvm/IR/CallingConv.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/DerivedTypes.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/OperandTraits.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/Use.h"
38#include "llvm/IR/User.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/ErrorHandling.h"
43#include <cassert>
44#include <cstddef>
45#include <cstdint>
46#include <iterator>
47
48namespace llvm {
49
50class APInt;
51class ConstantInt;
52class DataLayout;
53class LLVMContext;
54
55//===----------------------------------------------------------------------===//
56// AllocaInst Class
57//===----------------------------------------------------------------------===//
58
59/// an instruction to allocate memory on the stack
60class AllocaInst : public UnaryInstruction {
61 Type *AllocatedType;
62
63protected:
64 // Note: Instruction needs to be a friend here to call cloneImpl.
65 friend class Instruction;
66
67 AllocaInst *cloneImpl() const;
68
69public:
70 explicit AllocaInst(Type *Ty, unsigned AddrSpace,
71 Value *ArraySize = nullptr,
72 const Twine &Name = "",
73 Instruction *InsertBefore = nullptr);
74 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
75 const Twine &Name, BasicBlock *InsertAtEnd);
76
77 AllocaInst(Type *Ty, unsigned AddrSpace,
78 const Twine &Name, Instruction *InsertBefore = nullptr);
79 AllocaInst(Type *Ty, unsigned AddrSpace,
80 const Twine &Name, BasicBlock *InsertAtEnd);
81
82 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, unsigned Align,
83 const Twine &Name = "", Instruction *InsertBefore = nullptr);
84 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, unsigned Align,
85 const Twine &Name, BasicBlock *InsertAtEnd);
86
87 /// Return true if there is an allocation size parameter to the allocation
88 /// instruction that is not 1.
89 bool isArrayAllocation() const;
90
91 /// Get the number of elements allocated. For a simple allocation of a single
92 /// element, this will return a constant 1 value.
93 const Value *getArraySize() const { return getOperand(0); }
94 Value *getArraySize() { return getOperand(0); }
95
96 /// Overload to return most specific pointer type.
97 PointerType *getType() const {
98 return cast<PointerType>(Instruction::getType());
99 }
100
101 /// Return the type that is being allocated by the instruction.
102 Type *getAllocatedType() const { return AllocatedType; }
103 /// for use only in special circumstances that need to generically
104 /// transform a whole instruction (eg: IR linking and vectorization).
105 void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
106
107 /// Return the alignment of the memory that is being allocated by the
108 /// instruction.
109 unsigned getAlignment() const {
110 return (1u << (getSubclassDataFromInstruction() & 31)) >> 1;
111 }
112 void setAlignment(unsigned Align);
113
114 /// Return true if this alloca is in the entry block of the function and is a
115 /// constant size. If so, the code generator will fold it into the
116 /// prolog/epilog code, so it is basically free.
117 bool isStaticAlloca() const;
118
119 /// Return true if this alloca is used as an inalloca argument to a call. Such
120 /// allocas are never considered static even if they are in the entry block.
121 bool isUsedWithInAlloca() const {
122 return getSubclassDataFromInstruction() & 32;
123 }
124
125 /// Specify whether this alloca is used to represent the arguments to a call.
126 void setUsedWithInAlloca(bool V) {
127 setInstructionSubclassData((getSubclassDataFromInstruction() & ~32) |
128 (V ? 32 : 0));
129 }
130
131 /// Return true if this alloca is used as a swifterror argument to a call.
132 bool isSwiftError() const {
133 return getSubclassDataFromInstruction() & 64;
134 }
135
136 /// Specify whether this alloca is used to represent a swifterror.
137 void setSwiftError(bool V) {
138 setInstructionSubclassData((getSubclassDataFromInstruction() & ~64) |
139 (V ? 64 : 0));
140 }
141
142 // Methods for support type inquiry through isa, cast, and dyn_cast:
143 static bool classof(const Instruction *I) {
144 return (I->getOpcode() == Instruction::Alloca);
145 }
146 static bool classof(const Value *V) {
147 return isa<Instruction>(V) && classof(cast<Instruction>(V));
148 }
149
150private:
151 // Shadow Instruction::setInstructionSubclassData with a private forwarding
152 // method so that subclasses cannot accidentally use it.
153 void setInstructionSubclassData(unsigned short D) {
154 Instruction::setInstructionSubclassData(D);
155 }
156};
157
158//===----------------------------------------------------------------------===//
159// LoadInst Class
160//===----------------------------------------------------------------------===//
161
162/// An instruction for reading from memory. This uses the SubclassData field in
163/// Value to store whether or not the load is volatile.
164class LoadInst : public UnaryInstruction {
165 void AssertOK();
166
167protected:
168 // Note: Instruction needs to be a friend here to call cloneImpl.
169 friend class Instruction;
170
171 LoadInst *cloneImpl() const;
172
173public:
174 LoadInst(Value *Ptr, const Twine &NameStr, Instruction *InsertBefore);
175 LoadInst(Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
176 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile = false,
177 Instruction *InsertBefore = nullptr);
178 LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile = false,
179 Instruction *InsertBefore = nullptr)
180 : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
181 NameStr, isVolatile, InsertBefore) {}
182 LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
183 BasicBlock *InsertAtEnd);
184 LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
185 Instruction *InsertBefore = nullptr)
186 : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
187 NameStr, isVolatile, Align, InsertBefore) {}
188 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
189 unsigned Align, Instruction *InsertBefore = nullptr);
190 LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
191 unsigned Align, BasicBlock *InsertAtEnd);
192 LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
193 AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
194 Instruction *InsertBefore = nullptr)
195 : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
196 NameStr, isVolatile, Align, Order, SSID, InsertBefore) {}
197 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
198 unsigned Align, AtomicOrdering Order,
199 SyncScope::ID SSID = SyncScope::System,
200 Instruction *InsertBefore = nullptr);
201 LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
202 unsigned Align, AtomicOrdering Order, SyncScope::ID SSID,
203 BasicBlock *InsertAtEnd);
204 LoadInst(Value *Ptr, const char *NameStr, Instruction *InsertBefore);
205 LoadInst(Value *Ptr, const char *NameStr, BasicBlock *InsertAtEnd);
206 LoadInst(Type *Ty, Value *Ptr, const char *NameStr = nullptr,
207 bool isVolatile = false, Instruction *InsertBefore = nullptr);
208 explicit LoadInst(Value *Ptr, const char *NameStr = nullptr,
209 bool isVolatile = false,
210 Instruction *InsertBefore = nullptr)
211 : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
212 NameStr, isVolatile, InsertBefore) {}
213 LoadInst(Value *Ptr, const char *NameStr, bool isVolatile,
214 BasicBlock *InsertAtEnd);
215
216 /// Return true if this is a load from a volatile memory location.
217 bool isVolatile() const { return getSubclassDataFromInstruction() & 1; }
218
219 /// Specify whether this is a volatile load or not.
220 void setVolatile(bool V) {
221 setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
222 (V ? 1 : 0));
223 }
224
225 /// Return the alignment of the access that is being performed.
226 unsigned getAlignment() const {
227 return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1;
228 }
229
230 void setAlignment(unsigned Align);
231
232 /// Returns the ordering constraint of this load instruction.
233 AtomicOrdering getOrdering() const {
234 return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
235 }
236
237 /// Sets the ordering constraint of this load instruction. May not be Release
238 /// or AcquireRelease.
239 void setOrdering(AtomicOrdering Ordering) {
240 setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
241 ((unsigned)Ordering << 7));
242 }
243
244 /// Returns the synchronization scope ID of this load instruction.
245 SyncScope::ID getSyncScopeID() const {
246 return SSID;
247 }
248
249 /// Sets the synchronization scope ID of this load instruction.
250 void setSyncScopeID(SyncScope::ID SSID) {
251 this->SSID = SSID;
252 }
253
254 /// Sets the ordering constraint and the synchronization scope ID of this load
255 /// instruction.
256 void setAtomic(AtomicOrdering Ordering,
257 SyncScope::ID SSID = SyncScope::System) {
258 setOrdering(Ordering);
259 setSyncScopeID(SSID);
260 }
261
262 bool isSimple() const { return !isAtomic() && !isVolatile(); }
263
264 bool isUnordered() const {
265 return (getOrdering() == AtomicOrdering::NotAtomic ||
266 getOrdering() == AtomicOrdering::Unordered) &&
267 !isVolatile();
268 }
269
270 Value *getPointerOperand() { return getOperand(0); }
271 const Value *getPointerOperand() const { return getOperand(0); }
272 static unsigned getPointerOperandIndex() { return 0U; }
273 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
274
275 /// Returns the address space of the pointer operand.
276 unsigned getPointerAddressSpace() const {
277 return getPointerOperandType()->getPointerAddressSpace();
278 }
279
280 // Methods for support type inquiry through isa, cast, and dyn_cast:
281 static bool classof(const Instruction *I) {
282 return I->getOpcode() == Instruction::Load;
283 }
284 static bool classof(const Value *V) {
285 return isa<Instruction>(V) && classof(cast<Instruction>(V));
286 }
287
288private:
289 // Shadow Instruction::setInstructionSubclassData with a private forwarding
290 // method so that subclasses cannot accidentally use it.
291 void setInstructionSubclassData(unsigned short D) {
292 Instruction::setInstructionSubclassData(D);
293 }
294
295 /// The synchronization scope ID of this load instruction. Not quite enough
296 /// room in SubClassData for everything, so synchronization scope ID gets its
297 /// own field.
298 SyncScope::ID SSID;
299};
300
301//===----------------------------------------------------------------------===//
302// StoreInst Class
303//===----------------------------------------------------------------------===//
304
305/// An instruction for storing to memory.
306class StoreInst : public Instruction {
307 void AssertOK();
308
309protected:
310 // Note: Instruction needs to be a friend here to call cloneImpl.
311 friend class Instruction;
312
313 StoreInst *cloneImpl() const;
314
315public:
316 StoreInst(Value *Val, Value *Ptr, Instruction *InsertBefore);
317 StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
318 StoreInst(Value *Val, Value *Ptr, bool isVolatile = false,
319 Instruction *InsertBefore = nullptr);
320 StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
321 StoreInst(Value *Val, Value *Ptr, bool isVolatile,
322 unsigned Align, Instruction *InsertBefore = nullptr);
323 StoreInst(Value *Val, Value *Ptr, bool isVolatile,
324 unsigned Align, BasicBlock *InsertAtEnd);
325 StoreInst(Value *Val, Value *Ptr, bool isVolatile,
326 unsigned Align, AtomicOrdering Order,
327 SyncScope::ID SSID = SyncScope::System,
328 Instruction *InsertBefore = nullptr);
329 StoreInst(Value *Val, Value *Ptr, bool isVolatile,
330 unsigned Align, AtomicOrdering Order, SyncScope::ID SSID,
331 BasicBlock *InsertAtEnd);
332
333 // allocate space for exactly two operands
334 void *operator new(size_t s) {
335 return User::operator new(s, 2);
336 }
337
338 /// Return true if this is a store to a volatile memory location.
339 bool isVolatile() const { return getSubclassDataFromInstruction() & 1; }
340
341 /// Specify whether this is a volatile store or not.
342 void setVolatile(bool V) {
343 setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
344 (V ? 1 : 0));
345 }
346
347 /// Transparently provide more efficient getOperand methods.
348 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
349
350 /// Return the alignment of the access that is being performed
351 unsigned getAlignment() const {
352 return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1;
353 }
354
355 void setAlignment(unsigned Align);
356
357 /// Returns the ordering constraint of this store instruction.
358 AtomicOrdering getOrdering() const {
359 return AtomicOrdering((getSubclassDataFromInstruction() >> 7) & 7);
360 }
361
362 /// Sets the ordering constraint of this store instruction. May not be
363 /// Acquire or AcquireRelease.
364 void setOrdering(AtomicOrdering Ordering) {
365 setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) |
366 ((unsigned)Ordering << 7));
367 }
368
369 /// Returns the synchronization scope ID of this store instruction.
370 SyncScope::ID getSyncScopeID() const {
371 return SSID;
372 }
373
374 /// Sets the synchronization scope ID of this store instruction.
375 void setSyncScopeID(SyncScope::ID SSID) {
376 this->SSID = SSID;
377 }
378
379 /// Sets the ordering constraint and the synchronization scope ID of this
380 /// store instruction.
381 void setAtomic(AtomicOrdering Ordering,
382 SyncScope::ID SSID = SyncScope::System) {
383 setOrdering(Ordering);
384 setSyncScopeID(SSID);
385 }
386
387 bool isSimple() const { return !isAtomic() && !isVolatile(); }
388
389 bool isUnordered() const {
390 return (getOrdering() == AtomicOrdering::NotAtomic ||
391 getOrdering() == AtomicOrdering::Unordered) &&
392 !isVolatile();
393 }
394
395 Value *getValueOperand() { return getOperand(0); }
396 const Value *getValueOperand() const { return getOperand(0); }
397
398 Value *getPointerOperand() { return getOperand(1); }
399 const Value *getPointerOperand() const { return getOperand(1); }
400 static unsigned getPointerOperandIndex() { return 1U; }
401 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
402
403 /// Returns the address space of the pointer operand.
404 unsigned getPointerAddressSpace() const {
405 return getPointerOperandType()->getPointerAddressSpace();
406 }
407
408 // Methods for support type inquiry through isa, cast, and dyn_cast:
409 static bool classof(const Instruction *I) {
410 return I->getOpcode() == Instruction::Store;
411 }
412 static bool classof(const Value *V) {
413 return isa<Instruction>(V) && classof(cast<Instruction>(V));
414 }
415
416private:
417 // Shadow Instruction::setInstructionSubclassData with a private forwarding
418 // method so that subclasses cannot accidentally use it.
419 void setInstructionSubclassData(unsigned short D) {
420 Instruction::setInstructionSubclassData(D);
421 }
422
423 /// The synchronization scope ID of this store instruction. Not quite enough
424 /// room in SubClassData for everything, so synchronization scope ID gets its
425 /// own field.
426 SyncScope::ID SSID;
427};
428
429template <>
430struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
431};
432
433DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)StoreInst::op_iterator StoreInst::op_begin() { return OperandTraits
<StoreInst>::op_begin(this); } StoreInst::const_op_iterator
StoreInst::op_begin() const { return OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this)); } StoreInst
::op_iterator StoreInst::op_end() { return OperandTraits<StoreInst
>::op_end(this); } StoreInst::const_op_iterator StoreInst::
op_end() const { return OperandTraits<StoreInst>::op_end
(const_cast<StoreInst*>(this)); } Value *StoreInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<StoreInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 433, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<StoreInst>::op_begin(const_cast
<StoreInst*>(this))[i_nocapture].get()); } void StoreInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<StoreInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 433, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
StoreInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned StoreInst::getNumOperands() const { return OperandTraits
<StoreInst>::operands(this); } template <int Idx_nocapture
> Use &StoreInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
StoreInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
434
435//===----------------------------------------------------------------------===//
436// FenceInst Class
437//===----------------------------------------------------------------------===//
438
439/// An instruction for ordering other memory operations.
440class FenceInst : public Instruction {
441 void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
442
443protected:
444 // Note: Instruction needs to be a friend here to call cloneImpl.
445 friend class Instruction;
446
447 FenceInst *cloneImpl() const;
448
449public:
450 // Ordering may only be Acquire, Release, AcquireRelease, or
451 // SequentiallyConsistent.
452 FenceInst(LLVMContext &C, AtomicOrdering Ordering,
453 SyncScope::ID SSID = SyncScope::System,
454 Instruction *InsertBefore = nullptr);
455 FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
456 BasicBlock *InsertAtEnd);
457
458 // allocate space for exactly zero operands
459 void *operator new(size_t s) {
460 return User::operator new(s, 0);
461 }
462
463 /// Returns the ordering constraint of this fence instruction.
464 AtomicOrdering getOrdering() const {
465 return AtomicOrdering(getSubclassDataFromInstruction() >> 1);
466 }
467
468 /// Sets the ordering constraint of this fence instruction. May only be
469 /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
470 void setOrdering(AtomicOrdering Ordering) {
471 setInstructionSubclassData((getSubclassDataFromInstruction() & 1) |
472 ((unsigned)Ordering << 1));
473 }
474
475 /// Returns the synchronization scope ID of this fence instruction.
476 SyncScope::ID getSyncScopeID() const {
477 return SSID;
478 }
479
480 /// Sets the synchronization scope ID of this fence instruction.
481 void setSyncScopeID(SyncScope::ID SSID) {
482 this->SSID = SSID;
483 }
484
485 // Methods for support type inquiry through isa, cast, and dyn_cast:
486 static bool classof(const Instruction *I) {
487 return I->getOpcode() == Instruction::Fence;
488 }
489 static bool classof(const Value *V) {
490 return isa<Instruction>(V) && classof(cast<Instruction>(V));
491 }
492
493private:
494 // Shadow Instruction::setInstructionSubclassData with a private forwarding
495 // method so that subclasses cannot accidentally use it.
496 void setInstructionSubclassData(unsigned short D) {
497 Instruction::setInstructionSubclassData(D);
498 }
499
500 /// The synchronization scope ID of this fence instruction. Not quite enough
501 /// room in SubClassData for everything, so synchronization scope ID gets its
502 /// own field.
503 SyncScope::ID SSID;
504};
505
506//===----------------------------------------------------------------------===//
507// AtomicCmpXchgInst Class
508//===----------------------------------------------------------------------===//
509
510/// an instruction that atomically checks whether a
511/// specified value is in a memory location, and, if it is, stores a new value
512/// there. Returns the value that was loaded.
513///
514class AtomicCmpXchgInst : public Instruction {
515 void Init(Value *Ptr, Value *Cmp, Value *NewVal,
516 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
517 SyncScope::ID SSID);
518
519protected:
520 // Note: Instruction needs to be a friend here to call cloneImpl.
521 friend class Instruction;
522
523 AtomicCmpXchgInst *cloneImpl() const;
524
525public:
526 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
527 AtomicOrdering SuccessOrdering,
528 AtomicOrdering FailureOrdering,
529 SyncScope::ID SSID, Instruction *InsertBefore = nullptr);
530 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal,
531 AtomicOrdering SuccessOrdering,
532 AtomicOrdering FailureOrdering,
533 SyncScope::ID SSID, BasicBlock *InsertAtEnd);
534
535 // allocate space for exactly three operands
536 void *operator new(size_t s) {
537 return User::operator new(s, 3);
538 }
539
540 /// Return true if this is a cmpxchg from a volatile memory
541 /// location.
542 ///
543 bool isVolatile() const {
544 return getSubclassDataFromInstruction() & 1;
545 }
546
547 /// Specify whether this is a volatile cmpxchg.
548 ///
549 void setVolatile(bool V) {
550 setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
551 (unsigned)V);
552 }
553
554 /// Return true if this cmpxchg may spuriously fail.
555 bool isWeak() const {
556 return getSubclassDataFromInstruction() & 0x100;
557 }
558
559 void setWeak(bool IsWeak) {
560 setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x100) |
561 (IsWeak << 8));
562 }
563
564 /// Transparently provide more efficient getOperand methods.
565 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
566
567 /// Returns the success ordering constraint of this cmpxchg instruction.
568 AtomicOrdering getSuccessOrdering() const {
569 return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
570 }
571
572 /// Sets the success ordering constraint of this cmpxchg instruction.
573 void setSuccessOrdering(AtomicOrdering Ordering) {
574 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "CmpXchg instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"CmpXchg instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 575, __extension__ __PRETTY_FUNCTION__))
575 "CmpXchg instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "CmpXchg instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"CmpXchg instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 575, __extension__ __PRETTY_FUNCTION__));
576 setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x1c) |
577 ((unsigned)Ordering << 2));
578 }
579
580 /// Returns the failure ordering constraint of this cmpxchg instruction.
581 AtomicOrdering getFailureOrdering() const {
582 return AtomicOrdering((getSubclassDataFromInstruction() >> 5) & 7);
583 }
584
585 /// Sets the failure ordering constraint of this cmpxchg instruction.
586 void setFailureOrdering(AtomicOrdering Ordering) {
587 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "CmpXchg instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"CmpXchg instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 588, __extension__ __PRETTY_FUNCTION__))
588 "CmpXchg instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "CmpXchg instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"CmpXchg instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 588, __extension__ __PRETTY_FUNCTION__));
589 setInstructionSubclassData((getSubclassDataFromInstruction() & ~0xe0) |
590 ((unsigned)Ordering << 5));
591 }
592
593 /// Returns the synchronization scope ID of this cmpxchg instruction.
594 SyncScope::ID getSyncScopeID() const {
595 return SSID;
596 }
597
598 /// Sets the synchronization scope ID of this cmpxchg instruction.
599 void setSyncScopeID(SyncScope::ID SSID) {
600 this->SSID = SSID;
601 }
602
603 Value *getPointerOperand() { return getOperand(0); }
604 const Value *getPointerOperand() const { return getOperand(0); }
605 static unsigned getPointerOperandIndex() { return 0U; }
606
607 Value *getCompareOperand() { return getOperand(1); }
608 const Value *getCompareOperand() const { return getOperand(1); }
609
610 Value *getNewValOperand() { return getOperand(2); }
611 const Value *getNewValOperand() const { return getOperand(2); }
612
613 /// Returns the address space of the pointer operand.
614 unsigned getPointerAddressSpace() const {
615 return getPointerOperand()->getType()->getPointerAddressSpace();
616 }
617
618 /// Returns the strongest permitted ordering on failure, given the
619 /// desired ordering on success.
620 ///
621 /// If the comparison in a cmpxchg operation fails, there is no atomic store
622 /// so release semantics cannot be provided. So this function drops explicit
623 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
624 /// operation would remain SequentiallyConsistent.
625 static AtomicOrdering
626 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
627 switch (SuccessOrdering) {
628 default:
629 llvm_unreachable("invalid cmpxchg success ordering")::llvm::llvm_unreachable_internal("invalid cmpxchg success ordering"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 629);
630 case AtomicOrdering::Release:
631 case AtomicOrdering::Monotonic:
632 return AtomicOrdering::Monotonic;
633 case AtomicOrdering::AcquireRelease:
634 case AtomicOrdering::Acquire:
635 return AtomicOrdering::Acquire;
636 case AtomicOrdering::SequentiallyConsistent:
637 return AtomicOrdering::SequentiallyConsistent;
638 }
639 }
640
641 // Methods for support type inquiry through isa, cast, and dyn_cast:
642 static bool classof(const Instruction *I) {
643 return I->getOpcode() == Instruction::AtomicCmpXchg;
644 }
645 static bool classof(const Value *V) {
646 return isa<Instruction>(V) && classof(cast<Instruction>(V));
647 }
648
649private:
650 // Shadow Instruction::setInstructionSubclassData with a private forwarding
651 // method so that subclasses cannot accidentally use it.
652 void setInstructionSubclassData(unsigned short D) {
653 Instruction::setInstructionSubclassData(D);
654 }
655
656 /// The synchronization scope ID of this cmpxchg instruction. Not quite
657 /// enough room in SubClassData for everything, so synchronization scope ID
658 /// gets its own field.
659 SyncScope::ID SSID;
660};
661
662template <>
663struct OperandTraits<AtomicCmpXchgInst> :
664 public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
665};
666
667DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)AtomicCmpXchgInst::op_iterator AtomicCmpXchgInst::op_begin() {
return OperandTraits<AtomicCmpXchgInst>::op_begin(this
); } AtomicCmpXchgInst::const_op_iterator AtomicCmpXchgInst::
op_begin() const { return OperandTraits<AtomicCmpXchgInst>
::op_begin(const_cast<AtomicCmpXchgInst*>(this)); } AtomicCmpXchgInst
::op_iterator AtomicCmpXchgInst::op_end() { return OperandTraits
<AtomicCmpXchgInst>::op_end(this); } AtomicCmpXchgInst::
const_op_iterator AtomicCmpXchgInst::op_end() const { return OperandTraits
<AtomicCmpXchgInst>::op_end(const_cast<AtomicCmpXchgInst
*>(this)); } Value *AtomicCmpXchgInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 667, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<AtomicCmpXchgInst>::op_begin
(const_cast<AtomicCmpXchgInst*>(this))[i_nocapture].get
()); } void AtomicCmpXchgInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 667, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
AtomicCmpXchgInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned AtomicCmpXchgInst::getNumOperands() const { return
OperandTraits<AtomicCmpXchgInst>::operands(this); } template
<int Idx_nocapture> Use &AtomicCmpXchgInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &AtomicCmpXchgInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
668
669//===----------------------------------------------------------------------===//
670// AtomicRMWInst Class
671//===----------------------------------------------------------------------===//
672
673/// an instruction that atomically reads a memory location,
674/// combines it with another value, and then stores the result back. Returns
675/// the old value.
676///
677class AtomicRMWInst : public Instruction {
678protected:
679 // Note: Instruction needs to be a friend here to call cloneImpl.
680 friend class Instruction;
681
682 AtomicRMWInst *cloneImpl() const;
683
684public:
685 /// This enumeration lists the possible modifications atomicrmw can make. In
686 /// the descriptions, 'p' is the pointer to the instruction's memory location,
687 /// 'old' is the initial value of *p, and 'v' is the other value passed to the
688 /// instruction. These instructions always return 'old'.
689 enum BinOp {
690 /// *p = v
691 Xchg,
692 /// *p = old + v
693 Add,
694 /// *p = old - v
695 Sub,
696 /// *p = old & v
697 And,
698 /// *p = ~(old & v)
699 Nand,
700 /// *p = old | v
701 Or,
702 /// *p = old ^ v
703 Xor,
704 /// *p = old >signed v ? old : v
705 Max,
706 /// *p = old <signed v ? old : v
707 Min,
708 /// *p = old >unsigned v ? old : v
709 UMax,
710 /// *p = old <unsigned v ? old : v
711 UMin,
712
713 FIRST_BINOP = Xchg,
714 LAST_BINOP = UMin,
715 BAD_BINOP
716 };
717
718 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
719 AtomicOrdering Ordering, SyncScope::ID SSID,
720 Instruction *InsertBefore = nullptr);
721 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val,
722 AtomicOrdering Ordering, SyncScope::ID SSID,
723 BasicBlock *InsertAtEnd);
724
725 // allocate space for exactly two operands
726 void *operator new(size_t s) {
727 return User::operator new(s, 2);
728 }
729
730 BinOp getOperation() const {
731 return static_cast<BinOp>(getSubclassDataFromInstruction() >> 5);
732 }
733
734 void setOperation(BinOp Operation) {
735 unsigned short SubclassData = getSubclassDataFromInstruction();
736 setInstructionSubclassData((SubclassData & 31) |
737 (Operation << 5));
738 }
739
740 /// Return true if this is a RMW on a volatile memory location.
741 ///
742 bool isVolatile() const {
743 return getSubclassDataFromInstruction() & 1;
744 }
745
746 /// Specify whether this is a volatile RMW or not.
747 ///
748 void setVolatile(bool V) {
749 setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
750 (unsigned)V);
751 }
752
753 /// Transparently provide more efficient getOperand methods.
754 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
755
756 /// Returns the ordering constraint of this rmw instruction.
757 AtomicOrdering getOrdering() const {
758 return AtomicOrdering((getSubclassDataFromInstruction() >> 2) & 7);
759 }
760
761 /// Sets the ordering constraint of this rmw instruction.
762 void setOrdering(AtomicOrdering Ordering) {
763 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 764, __extension__ __PRETTY_FUNCTION__))
764 "atomicrmw instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 764, __extension__ __PRETTY_FUNCTION__));
765 setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 2)) |
766 ((unsigned)Ordering << 2));
767 }
768
769 /// Returns the synchronization scope ID of this rmw instruction.
770 SyncScope::ID getSyncScopeID() const {
771 return SSID;
772 }
773
774 /// Sets the synchronization scope ID of this rmw instruction.
775 void setSyncScopeID(SyncScope::ID SSID) {
776 this->SSID = SSID;
777 }
778
779 Value *getPointerOperand() { return getOperand(0); }
780 const Value *getPointerOperand() const { return getOperand(0); }
781 static unsigned getPointerOperandIndex() { return 0U; }
782
783 Value *getValOperand() { return getOperand(1); }
784 const Value *getValOperand() const { return getOperand(1); }
785
786 /// Returns the address space of the pointer operand.
787 unsigned getPointerAddressSpace() const {
788 return getPointerOperand()->getType()->getPointerAddressSpace();
789 }
790
791 // Methods for support type inquiry through isa, cast, and dyn_cast:
792 static bool classof(const Instruction *I) {
793 return I->getOpcode() == Instruction::AtomicRMW;
794 }
795 static bool classof(const Value *V) {
796 return isa<Instruction>(V) && classof(cast<Instruction>(V));
797 }
798
799private:
800 void Init(BinOp Operation, Value *Ptr, Value *Val,
801 AtomicOrdering Ordering, SyncScope::ID SSID);
802
803 // Shadow Instruction::setInstructionSubclassData with a private forwarding
804 // method so that subclasses cannot accidentally use it.
805 void setInstructionSubclassData(unsigned short D) {
806 Instruction::setInstructionSubclassData(D);
807 }
808
809 /// The synchronization scope ID of this rmw instruction. Not quite enough
810 /// room in SubClassData for everything, so synchronization scope ID gets its
811 /// own field.
812 SyncScope::ID SSID;
813};
814
815template <>
816struct OperandTraits<AtomicRMWInst>
817 : public FixedNumOperandTraits<AtomicRMWInst,2> {
818};
819
820DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)AtomicRMWInst::op_iterator AtomicRMWInst::op_begin() { return
OperandTraits<AtomicRMWInst>::op_begin(this); } AtomicRMWInst
::const_op_iterator AtomicRMWInst::op_begin() const { return OperandTraits
<AtomicRMWInst>::op_begin(const_cast<AtomicRMWInst*>
(this)); } AtomicRMWInst::op_iterator AtomicRMWInst::op_end()
{ return OperandTraits<AtomicRMWInst>::op_end(this); }
AtomicRMWInst::const_op_iterator AtomicRMWInst::op_end() const
{ return OperandTraits<AtomicRMWInst>::op_end(const_cast
<AtomicRMWInst*>(this)); } Value *AtomicRMWInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 820, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<AtomicRMWInst>::op_begin(const_cast
<AtomicRMWInst*>(this))[i_nocapture].get()); } void AtomicRMWInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<AtomicRMWInst
>::operands(this) && "setOperand() out of range!")
? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 820, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
AtomicRMWInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned AtomicRMWInst::getNumOperands() const { return OperandTraits
<AtomicRMWInst>::operands(this); } template <int Idx_nocapture
> Use &AtomicRMWInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &AtomicRMWInst::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
821
822//===----------------------------------------------------------------------===//
823// GetElementPtrInst Class
824//===----------------------------------------------------------------------===//
825
826// checkGEPType - Simple wrapper function to give a better assertion failure
827// message on bad indexes for a gep instruction.
828//
829inline Type *checkGEPType(Type *Ty) {
830 assert(Ty && "Invalid GetElementPtrInst indices for type!")(static_cast <bool> (Ty && "Invalid GetElementPtrInst indices for type!"
) ? void (0) : __assert_fail ("Ty && \"Invalid GetElementPtrInst indices for type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 830, __extension__ __PRETTY_FUNCTION__));
831 return Ty;
832}
833
834/// an instruction for type-safe pointer arithmetic to
835/// access elements of arrays and structs
836///
837class GetElementPtrInst : public Instruction {
838 Type *SourceElementType;
839 Type *ResultElementType;
840
841 GetElementPtrInst(const GetElementPtrInst &GEPI);
842
843 /// Constructors - Create a getelementptr instruction with a base pointer an
844 /// list of indices. The first ctor can optionally insert before an existing
845 /// instruction, the second appends the new instruction to the specified
846 /// BasicBlock.
847 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
848 ArrayRef<Value *> IdxList, unsigned Values,
849 const Twine &NameStr, Instruction *InsertBefore);
850 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
851 ArrayRef<Value *> IdxList, unsigned Values,
852 const Twine &NameStr, BasicBlock *InsertAtEnd);
853
854 void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
855
856protected:
857 // Note: Instruction needs to be a friend here to call cloneImpl.
858 friend class Instruction;
859
860 GetElementPtrInst *cloneImpl() const;
861
862public:
863 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
864 ArrayRef<Value *> IdxList,
865 const Twine &NameStr = "",
866 Instruction *InsertBefore = nullptr) {
867 unsigned Values = 1 + unsigned(IdxList.size());
868 if (!PointeeType)
869 PointeeType =
870 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType();
871 else
872 assert((static_cast <bool> (PointeeType == cast<PointerType
>(Ptr->getType()->getScalarType())->getElementType
()) ? void (0) : __assert_fail ("PointeeType == cast<PointerType>(Ptr->getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 874, __extension__ __PRETTY_FUNCTION__))
873 PointeeType ==(static_cast <bool> (PointeeType == cast<PointerType
>(Ptr->getType()->getScalarType())->getElementType
()) ? void (0) : __assert_fail ("PointeeType == cast<PointerType>(Ptr->getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 874, __extension__ __PRETTY_FUNCTION__))
874 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType())(static_cast <bool> (PointeeType == cast<PointerType
>(Ptr->getType()->getScalarType())->getElementType
()) ? void (0) : __assert_fail ("PointeeType == cast<PointerType>(Ptr->getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 874, __extension__ __PRETTY_FUNCTION__));
875 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
876 NameStr, InsertBefore);
877 }
878
879 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
880 ArrayRef<Value *> IdxList,
881 const Twine &NameStr,
882 BasicBlock *InsertAtEnd) {
883 unsigned Values = 1 + unsigned(IdxList.size());
884 if (!PointeeType)
885 PointeeType =
886 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType();
887 else
888 assert((static_cast <bool> (PointeeType == cast<PointerType
>(Ptr->getType()->getScalarType())->getElementType
()) ? void (0) : __assert_fail ("PointeeType == cast<PointerType>(Ptr->getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 890, __extension__ __PRETTY_FUNCTION__))
889 PointeeType ==(static_cast <bool> (PointeeType == cast<PointerType
>(Ptr->getType()->getScalarType())->getElementType
()) ? void (0) : __assert_fail ("PointeeType == cast<PointerType>(Ptr->getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 890, __extension__ __PRETTY_FUNCTION__))
890 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType())(static_cast <bool> (PointeeType == cast<PointerType
>(Ptr->getType()->getScalarType())->getElementType
()) ? void (0) : __assert_fail ("PointeeType == cast<PointerType>(Ptr->getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 890, __extension__ __PRETTY_FUNCTION__));
891 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
892 NameStr, InsertAtEnd);
893 }
894
895 /// Create an "inbounds" getelementptr. See the documentation for the
896 /// "inbounds" flag in LangRef.html for details.
897 static GetElementPtrInst *CreateInBounds(Value *Ptr,
898 ArrayRef<Value *> IdxList,
899 const Twine &NameStr = "",
900 Instruction *InsertBefore = nullptr){
901 return CreateInBounds(nullptr, Ptr, IdxList, NameStr, InsertBefore);
902 }
903
904 static GetElementPtrInst *
905 CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef<Value *> IdxList,
906 const Twine &NameStr = "",
907 Instruction *InsertBefore = nullptr) {
908 GetElementPtrInst *GEP =
909 Create(PointeeType, Ptr, IdxList, NameStr, InsertBefore);
910 GEP->setIsInBounds(true);
911 return GEP;
912 }
913
914 static GetElementPtrInst *CreateInBounds(Value *Ptr,
915 ArrayRef<Value *> IdxList,
916 const Twine &NameStr,
917 BasicBlock *InsertAtEnd) {
918 return CreateInBounds(nullptr, Ptr, IdxList, NameStr, InsertAtEnd);
919 }
920
921 static GetElementPtrInst *CreateInBounds(Type *PointeeType, Value *Ptr,
922 ArrayRef<Value *> IdxList,
923 const Twine &NameStr,
924 BasicBlock *InsertAtEnd) {
925 GetElementPtrInst *GEP =
926 Create(PointeeType, Ptr, IdxList, NameStr, InsertAtEnd);
927 GEP->setIsInBounds(true);
928 return GEP;
929 }
930
931 /// Transparently provide more efficient getOperand methods.
932 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
933
934 Type *getSourceElementType() const { return SourceElementType; }
935
936 void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
937 void setResultElementType(Type *Ty) { ResultElementType = Ty; }
938
939 Type *getResultElementType() const {
940 assert(ResultElementType ==(static_cast <bool> (ResultElementType == cast<PointerType
>(getType()->getScalarType())->getElementType()) ? void
(0) : __assert_fail ("ResultElementType == cast<PointerType>(getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 941, __extension__ __PRETTY_FUNCTION__))
941 cast<PointerType>(getType()->getScalarType())->getElementType())(static_cast <bool> (ResultElementType == cast<PointerType
>(getType()->getScalarType())->getElementType()) ? void
(0) : __assert_fail ("ResultElementType == cast<PointerType>(getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 941, __extension__ __PRETTY_FUNCTION__));
942 return ResultElementType;
943 }
944
945 /// Returns the address space of this instruction's pointer type.
946 unsigned getAddressSpace() const {
947 // Note that this is always the same as the pointer operand's address space
948 // and that is cheaper to compute, so cheat here.
949 return getPointerAddressSpace();
950 }
951
952 /// Returns the type of the element that would be loaded with
953 /// a load instruction with the specified parameters.
954 ///
955 /// Null is returned if the indices are invalid for the specified
956 /// pointer type.
957 ///
958 static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
959 static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
960 static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
961
962 inline op_iterator idx_begin() { return op_begin()+1; }
963 inline const_op_iterator idx_begin() const { return op_begin()+1; }
964 inline op_iterator idx_end() { return op_end(); }
965 inline const_op_iterator idx_end() const { return op_end(); }
966
967 inline iterator_range<op_iterator> indices() {
968 return make_range(idx_begin(), idx_end());
969 }
970
971 inline iterator_range<const_op_iterator> indices() const {
972 return make_range(idx_begin(), idx_end());
973 }
974
975 Value *getPointerOperand() {
976 return getOperand(0);
977 }
978 const Value *getPointerOperand() const {
979 return getOperand(0);
980 }
981 static unsigned getPointerOperandIndex() {
982 return 0U; // get index for modifying correct operand.
983 }
984
985 /// Method to return the pointer operand as a
986 /// PointerType.
987 Type *getPointerOperandType() const {
988 return getPointerOperand()->getType();
989 }
990
991 /// Returns the address space of the pointer operand.
992 unsigned getPointerAddressSpace() const {
993 return getPointerOperandType()->getPointerAddressSpace();
994 }
995
996 /// Returns the pointer type returned by the GEP
997 /// instruction, which may be a vector of pointers.
998 static Type *getGEPReturnType(Value *Ptr, ArrayRef<Value *> IdxList) {
999 return getGEPReturnType(
1000 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType(),
1001 Ptr, IdxList);
1002 }
1003 static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
1004 ArrayRef<Value *> IdxList) {
1005 Type *PtrTy = PointerType::get(checkGEPType(getIndexedType(ElTy, IdxList)),
1006 Ptr->getType()->getPointerAddressSpace());
1007 // Vector GEP
1008 if (Ptr->getType()->isVectorTy()) {
1009 unsigned NumElem = Ptr->getType()->getVectorNumElements();
1010 return VectorType::get(PtrTy, NumElem);
1011 }
1012 for (Value *Index : IdxList)
1013 if (Index->getType()->isVectorTy()) {
1014 unsigned NumElem = Index->getType()->getVectorNumElements();
1015 return VectorType::get(PtrTy, NumElem);
1016 }
1017 // Scalar GEP
1018 return PtrTy;
1019 }
1020
1021 unsigned getNumIndices() const { // Note: always non-negative
1022 return getNumOperands() - 1;
1023 }
1024
1025 bool hasIndices() const {
1026 return getNumOperands() > 1;
1027 }
1028
1029 /// Return true if all of the indices of this GEP are
1030 /// zeros. If so, the result pointer and the first operand have the same
1031 /// value, just potentially different types.
1032 bool hasAllZeroIndices() const;
1033
1034 /// Return true if all of the indices of this GEP are
1035 /// constant integers. If so, the result pointer and the first operand have
1036 /// a constant offset between them.
1037 bool hasAllConstantIndices() const;
1038
1039 /// Set or clear the inbounds flag on this GEP instruction.
1040 /// See LangRef.html for the meaning of inbounds on a getelementptr.
1041 void setIsInBounds(bool b = true);
1042
1043 /// Determine whether the GEP has the inbounds flag.
1044 bool isInBounds() const;
1045
1046 /// Accumulate the constant address offset of this GEP if possible.
1047 ///
1048 /// This routine accepts an APInt into which it will accumulate the constant
1049 /// offset of this GEP if the GEP is in fact constant. If the GEP is not
1050 /// all-constant, it returns false and the value of the offset APInt is
1051 /// undefined (it is *not* preserved!). The APInt passed into this routine
1052 /// must be at least as wide as the IntPtr type for the address space of
1053 /// the base GEP pointer.
1054 bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
1055
1056 // Methods for support type inquiry through isa, cast, and dyn_cast:
1057 static bool classof(const Instruction *I) {
1058 return (I->getOpcode() == Instruction::GetElementPtr);
1059 }
1060 static bool classof(const Value *V) {
1061 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1062 }
1063};
1064
1065template <>
1066struct OperandTraits<GetElementPtrInst> :
1067 public VariadicOperandTraits<GetElementPtrInst, 1> {
1068};
1069
1070GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1071 ArrayRef<Value *> IdxList, unsigned Values,
1072 const Twine &NameStr,
1073 Instruction *InsertBefore)
1074 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1075 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1076 Values, InsertBefore),
1077 SourceElementType(PointeeType),
1078 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1079 assert(ResultElementType ==(static_cast <bool> (ResultElementType == cast<PointerType
>(getType()->getScalarType())->getElementType()) ? void
(0) : __assert_fail ("ResultElementType == cast<PointerType>(getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1080, __extension__ __PRETTY_FUNCTION__))
1080 cast<PointerType>(getType()->getScalarType())->getElementType())(static_cast <bool> (ResultElementType == cast<PointerType
>(getType()->getScalarType())->getElementType()) ? void
(0) : __assert_fail ("ResultElementType == cast<PointerType>(getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1080, __extension__ __PRETTY_FUNCTION__));
1081 init(Ptr, IdxList, NameStr);
1082}
1083
1084GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1085 ArrayRef<Value *> IdxList, unsigned Values,
1086 const Twine &NameStr,
1087 BasicBlock *InsertAtEnd)
1088 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1089 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1090 Values, InsertAtEnd),
1091 SourceElementType(PointeeType),
1092 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1093 assert(ResultElementType ==(static_cast <bool> (ResultElementType == cast<PointerType
>(getType()->getScalarType())->getElementType()) ? void
(0) : __assert_fail ("ResultElementType == cast<PointerType>(getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1094, __extension__ __PRETTY_FUNCTION__))
1094 cast<PointerType>(getType()->getScalarType())->getElementType())(static_cast <bool> (ResultElementType == cast<PointerType
>(getType()->getScalarType())->getElementType()) ? void
(0) : __assert_fail ("ResultElementType == cast<PointerType>(getType()->getScalarType())->getElementType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1094, __extension__ __PRETTY_FUNCTION__));
1095 init(Ptr, IdxList, NameStr);
1096}
1097
1098DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)GetElementPtrInst::op_iterator GetElementPtrInst::op_begin() {
return OperandTraits<GetElementPtrInst>::op_begin(this
); } GetElementPtrInst::const_op_iterator GetElementPtrInst::
op_begin() const { return OperandTraits<GetElementPtrInst>
::op_begin(const_cast<GetElementPtrInst*>(this)); } GetElementPtrInst
::op_iterator GetElementPtrInst::op_end() { return OperandTraits
<GetElementPtrInst>::op_end(this); } GetElementPtrInst::
const_op_iterator GetElementPtrInst::op_end() const { return OperandTraits
<GetElementPtrInst>::op_end(const_cast<GetElementPtrInst
*>(this)); } Value *GetElementPtrInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<GetElementPtrInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1098, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<GetElementPtrInst>::op_begin
(const_cast<GetElementPtrInst*>(this))[i_nocapture].get
()); } void GetElementPtrInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<GetElementPtrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1098, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
GetElementPtrInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned GetElementPtrInst::getNumOperands() const { return
OperandTraits<GetElementPtrInst>::operands(this); } template
<int Idx_nocapture> Use &GetElementPtrInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &GetElementPtrInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1099
1100//===----------------------------------------------------------------------===//
1101// ICmpInst Class
1102//===----------------------------------------------------------------------===//
1103
1104/// This instruction compares its operands according to the predicate given
1105/// to the constructor. It only operates on integers or pointers. The operands
1106/// must be identical types.
1107/// Represent an integer comparison operator.
1108class ICmpInst: public CmpInst {
1109 void AssertOK() {
1110 assert(isIntPredicate() &&(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1111, __extension__ __PRETTY_FUNCTION__))
1111 "Invalid ICmp predicate value")(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1111, __extension__ __PRETTY_FUNCTION__));
1112 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1113, __extension__ __PRETTY_FUNCTION__))
1113 "Both operands to ICmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1113, __extension__ __PRETTY_FUNCTION__));
1114 // Check that the operands are the right type
1115 assert((getOperand(0)->getType()->isIntOrIntVectorTy() ||(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1117, __extension__ __PRETTY_FUNCTION__))
1116 getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1117, __extension__ __PRETTY_FUNCTION__))
1117 "Invalid operand types for ICmp instruction")(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1117, __extension__ __PRETTY_FUNCTION__));
1118 }
1119
1120protected:
1121 // Note: Instruction needs to be a friend here to call cloneImpl.
1122 friend class Instruction;
1123
1124 /// Clone an identical ICmpInst
1125 ICmpInst *cloneImpl() const;
1126
1127public:
1128 /// Constructor with insert-before-instruction semantics.
1129 ICmpInst(
1130 Instruction *InsertBefore, ///< Where to insert
1131 Predicate pred, ///< The predicate to use for the comparison
1132 Value *LHS, ///< The left-hand-side of the expression
1133 Value *RHS, ///< The right-hand-side of the expression
1134 const Twine &NameStr = "" ///< Name of the instruction
1135 ) : CmpInst(makeCmpResultType(LHS->getType()),
1136 Instruction::ICmp, pred, LHS, RHS, NameStr,
1137 InsertBefore) {
1138#ifndef NDEBUG
1139 AssertOK();
1140#endif
1141 }
1142
1143 /// Constructor with insert-at-end semantics.
1144 ICmpInst(
1145 BasicBlock &InsertAtEnd, ///< Block to insert into.
1146 Predicate pred, ///< The predicate to use for the comparison
1147 Value *LHS, ///< The left-hand-side of the expression
1148 Value *RHS, ///< The right-hand-side of the expression
1149 const Twine &NameStr = "" ///< Name of the instruction
1150 ) : CmpInst(makeCmpResultType(LHS->getType()),
1151 Instruction::ICmp, pred, LHS, RHS, NameStr,
1152 &InsertAtEnd) {
1153#ifndef NDEBUG
1154 AssertOK();
1155#endif
1156 }
1157
1158 /// Constructor with no-insertion semantics
1159 ICmpInst(
1160 Predicate pred, ///< The predicate to use for the comparison
1161 Value *LHS, ///< The left-hand-side of the expression
1162 Value *RHS, ///< The right-hand-side of the expression
1163 const Twine &NameStr = "" ///< Name of the instruction
1164 ) : CmpInst(makeCmpResultType(LHS->getType()),
1165 Instruction::ICmp, pred, LHS, RHS, NameStr) {
1166#ifndef NDEBUG
1167 AssertOK();
1168#endif
1169 }
1170
1171 /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
1172 /// @returns the predicate that would be the result if the operand were
1173 /// regarded as signed.
1174 /// Return the signed version of the predicate
1175 Predicate getSignedPredicate() const {
1176 return getSignedPredicate(getPredicate());
1177 }
1178
1179 /// This is a static version that you can use without an instruction.
1180 /// Return the signed version of the predicate.
1181 static Predicate getSignedPredicate(Predicate pred);
1182
1183 /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
1184 /// @returns the predicate that would be the result if the operand were
1185 /// regarded as unsigned.
1186 /// Return the unsigned version of the predicate
1187 Predicate getUnsignedPredicate() const {
1188 return getUnsignedPredicate(getPredicate());
1189 }
1190
1191 /// This is a static version that you can use without an instruction.
1192 /// Return the unsigned version of the predicate.
1193 static Predicate getUnsignedPredicate(Predicate pred);
1194
1195 /// Return true if this predicate is either EQ or NE. This also
1196 /// tests for commutativity.
1197 static bool isEquality(Predicate P) {
1198 return P == ICMP_EQ || P == ICMP_NE;
1199 }
1200
1201 /// Return true if this predicate is either EQ or NE. This also
1202 /// tests for commutativity.
1203 bool isEquality() const {
1204 return isEquality(getPredicate());
1205 }
1206
1207 /// @returns true if the predicate of this ICmpInst is commutative
1208 /// Determine if this relation is commutative.
1209 bool isCommutative() const { return isEquality(); }
1210
1211 /// Return true if the predicate is relational (not EQ or NE).
1212 ///
1213 bool isRelational() const {
1214 return !isEquality();
1215 }
1216
1217 /// Return true if the predicate is relational (not EQ or NE).
1218 ///
1219 static bool isRelational(Predicate P) {
1220 return !isEquality(P);
1221 }
1222
1223 /// Exchange the two operands to this instruction in such a way that it does
1224 /// not modify the semantics of the instruction. The predicate value may be
1225 /// changed to retain the same result if the predicate is order dependent
1226 /// (e.g. ult).
1227 /// Swap operands and adjust predicate.
1228 void swapOperands() {
1229 setPredicate(getSwappedPredicate());
1230 Op<0>().swap(Op<1>());
1231 }
1232
1233 // Methods for support type inquiry through isa, cast, and dyn_cast:
1234 static bool classof(const Instruction *I) {
1235 return I->getOpcode() == Instruction::ICmp;
1236 }
1237 static bool classof(const Value *V) {
1238 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1239 }
1240};
1241
1242//===----------------------------------------------------------------------===//
1243// FCmpInst Class
1244//===----------------------------------------------------------------------===//
1245
1246/// This instruction compares its operands according to the predicate given
1247/// to the constructor. It only operates on floating point values or packed
1248/// vectors of floating point values. The operands must be identical types.
1249/// Represents a floating point comparison operator.
1250class FCmpInst: public CmpInst {
1251 void AssertOK() {
1252 assert(isFPPredicate() && "Invalid FCmp predicate value")(static_cast <bool> (isFPPredicate() && "Invalid FCmp predicate value"
) ? void (0) : __assert_fail ("isFPPredicate() && \"Invalid FCmp predicate value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1252, __extension__ __PRETTY_FUNCTION__));
1253 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1254, __extension__ __PRETTY_FUNCTION__))
1254 "Both operands to FCmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1254, __extension__ __PRETTY_FUNCTION__));
1255 // Check that the operands are the right type
1256 assert(getOperand(0)->getType()->isFPOrFPVectorTy() &&(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1257, __extension__ __PRETTY_FUNCTION__))
1257 "Invalid operand types for FCmp instruction")(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1257, __extension__ __PRETTY_FUNCTION__));
1258 }
1259
1260protected:
1261 // Note: Instruction needs to be a friend here to call cloneImpl.
1262 friend class Instruction;
1263
1264 /// Clone an identical FCmpInst
1265 FCmpInst *cloneImpl() const;
1266
1267public:
1268 /// Constructor with insert-before-instruction semantics.
1269 FCmpInst(
1270 Instruction *InsertBefore, ///< Where to insert
1271 Predicate pred, ///< The predicate to use for the comparison
1272 Value *LHS, ///< The left-hand-side of the expression
1273 Value *RHS, ///< The right-hand-side of the expression
1274 const Twine &NameStr = "" ///< Name of the instruction
1275 ) : CmpInst(makeCmpResultType(LHS->getType()),
1276 Instruction::FCmp, pred, LHS, RHS, NameStr,
1277 InsertBefore) {
1278 AssertOK();
1279 }
1280
1281 /// Constructor with insert-at-end semantics.
1282 FCmpInst(
1283 BasicBlock &InsertAtEnd, ///< Block to insert into.
1284 Predicate pred, ///< The predicate to use for the comparison
1285 Value *LHS, ///< The left-hand-side of the expression
1286 Value *RHS, ///< The right-hand-side of the expression
1287 const Twine &NameStr = "" ///< Name of the instruction
1288 ) : CmpInst(makeCmpResultType(LHS->getType()),
1289 Instruction::FCmp, pred, LHS, RHS, NameStr,
1290 &InsertAtEnd) {
1291 AssertOK();
1292 }
1293
1294 /// Constructor with no-insertion semantics
1295 FCmpInst(
1296 Predicate pred, ///< The predicate to use for the comparison
1297 Value *LHS, ///< The left-hand-side of the expression
1298 Value *RHS, ///< The right-hand-side of the expression
1299 const Twine &NameStr = "" ///< Name of the instruction
1300 ) : CmpInst(makeCmpResultType(LHS->getType()),
1301 Instruction::FCmp, pred, LHS, RHS, NameStr) {
1302 AssertOK();
1303 }
1304
1305 /// @returns true if the predicate of this instruction is EQ or NE.
1306 /// Determine if this is an equality predicate.
1307 static bool isEquality(Predicate Pred) {
1308 return Pred == FCMP_OEQ || Pred == FCMP_ONE || Pred == FCMP_UEQ ||
1309 Pred == FCMP_UNE;
1310 }
1311
1312 /// @returns true if the predicate of this instruction is EQ or NE.
1313 /// Determine if this is an equality predicate.
1314 bool isEquality() const { return isEquality(getPredicate()); }
1315
1316 /// @returns true if the predicate of this instruction is commutative.
1317 /// Determine if this is a commutative predicate.
1318 bool isCommutative() const {
1319 return isEquality() ||
1320 getPredicate() == FCMP_FALSE ||
1321 getPredicate() == FCMP_TRUE ||
1322 getPredicate() == FCMP_ORD ||
1323 getPredicate() == FCMP_UNO;
1324 }
1325
1326 /// @returns true if the predicate is relational (not EQ or NE).
1327 /// Determine if this a relational predicate.
1328 bool isRelational() const { return !isEquality(); }
1329
1330 /// Exchange the two operands to this instruction in such a way that it does
1331 /// not modify the semantics of the instruction. The predicate value may be
1332 /// changed to retain the same result if the predicate is order dependent
1333 /// (e.g. ult).
1334 /// Swap operands and adjust predicate.
1335 void swapOperands() {
1336 setPredicate(getSwappedPredicate());
1337 Op<0>().swap(Op<1>());
1338 }
1339
1340 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1341 static bool classof(const Instruction *I) {
1342 return I->getOpcode() == Instruction::FCmp;
1343 }
1344 static bool classof(const Value *V) {
1345 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1346 }
1347};
1348
1349//===----------------------------------------------------------------------===//
1350/// This class represents a function call, abstracting a target
1351/// machine's calling convention. This class uses low bit of the SubClassData
1352/// field to indicate whether or not this is a tail call. The rest of the bits
1353/// hold the calling convention of the call.
1354///
1355class CallInst : public Instruction,
1356 public OperandBundleUser<CallInst, User::op_iterator> {
1357 friend class OperandBundleUser<CallInst, User::op_iterator>;
1358
1359 AttributeList Attrs; ///< parameter attributes for call
1360 FunctionType *FTy;
1361
1362 CallInst(const CallInst &CI);
1363
1364 /// Construct a CallInst given a range of arguments.
1365 /// Construct a CallInst from a range of arguments
1366 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1367 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1368 Instruction *InsertBefore);
1369
1370 inline CallInst(Value *Func, ArrayRef<Value *> Args,
1371 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1372 Instruction *InsertBefore)
1373 : CallInst(cast<FunctionType>(
1374 cast<PointerType>(Func->getType())->getElementType()),
1375 Func, Args, Bundles, NameStr, InsertBefore) {}
1376
1377 inline CallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr,
1378 Instruction *InsertBefore)
1379 : CallInst(Func, Args, None, NameStr, InsertBefore) {}
1380
1381 /// Construct a CallInst given a range of arguments.
1382 /// Construct a CallInst from a range of arguments
1383 inline CallInst(Value *Func, ArrayRef<Value *> Args,
1384 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1385 BasicBlock *InsertAtEnd);
1386
1387 explicit CallInst(Value *F, const Twine &NameStr,
1388 Instruction *InsertBefore);
1389
1390 CallInst(Value *F, const Twine &NameStr, BasicBlock *InsertAtEnd);
1391
1392 void init(Value *Func, ArrayRef<Value *> Args,
1393 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr) {
1394 init(cast<FunctionType>(
1395 cast<PointerType>(Func->getType())->getElementType()),
1396 Func, Args, Bundles, NameStr);
1397 }
1398 void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
1399 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
1400 void init(Value *Func, const Twine &NameStr);
1401
1402 bool hasDescriptor() const { return HasDescriptor; }
1403
1404protected:
1405 // Note: Instruction needs to be a friend here to call cloneImpl.
1406 friend class Instruction;
1407
1408 CallInst *cloneImpl() const;
1409
1410public:
1411 static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
1412 ArrayRef<OperandBundleDef> Bundles = None,
1413 const Twine &NameStr = "",
1414 Instruction *InsertBefore = nullptr) {
1415 return Create(cast<FunctionType>(
1416 cast<PointerType>(Func->getType())->getElementType()),
1417 Func, Args, Bundles, NameStr, InsertBefore);
1418 }
1419
1420 static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
1421 const Twine &NameStr,
1422 Instruction *InsertBefore = nullptr) {
1423 return Create(cast<FunctionType>(
1424 cast<PointerType>(Func->getType())->getElementType()),
1425 Func, Args, None, NameStr, InsertBefore);
1426 }
1427
1428 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1429 const Twine &NameStr,
1430 Instruction *InsertBefore = nullptr) {
1431 return new (unsigned(Args.size() + 1))
1432 CallInst(Ty, Func, Args, None, NameStr, InsertBefore);
1433 }
1434
1435 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1436 ArrayRef<OperandBundleDef> Bundles = None,
1437 const Twine &NameStr = "",
1438 Instruction *InsertBefore = nullptr) {
1439 const unsigned TotalOps =
1440 unsigned(Args.size()) + CountBundleInputs(Bundles) + 1;
1441 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1442
1443 return new (TotalOps, DescriptorBytes)
1444 CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
1445 }
1446
1447 static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
1448 ArrayRef<OperandBundleDef> Bundles,
1449 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1450 const unsigned TotalOps =
1451 unsigned(Args.size()) + CountBundleInputs(Bundles) + 1;
1452 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1453
1454 return new (TotalOps, DescriptorBytes)
1455 CallInst(Func, Args, Bundles, NameStr, InsertAtEnd);
1456 }
1457
1458 static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
1459 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1460 return new (unsigned(Args.size() + 1))
1461 CallInst(Func, Args, None, NameStr, InsertAtEnd);
1462 }
1463
1464 static CallInst *Create(Value *F, const Twine &NameStr = "",
1465 Instruction *InsertBefore = nullptr) {
1466 return new(1) CallInst(F, NameStr, InsertBefore);
1467 }
1468
1469 static CallInst *Create(Value *F, const Twine &NameStr,
1470 BasicBlock *InsertAtEnd) {
1471 return new(1) CallInst(F, NameStr, InsertAtEnd);
1472 }
1473
1474 /// Create a clone of \p CI with a different set of operand bundles and
1475 /// insert it before \p InsertPt.
1476 ///
1477 /// The returned call instruction is identical \p CI in every way except that
1478 /// the operand bundles for the new instruction are set to the operand bundles
1479 /// in \p Bundles.
1480 static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
1481 Instruction *InsertPt = nullptr);
1482
1483 /// Generate the IR for a call to malloc:
1484 /// 1. Compute the malloc call's argument as the specified type's size,
1485 /// possibly multiplied by the array size if the array size is not
1486 /// constant 1.
1487 /// 2. Call malloc with that argument.
1488 /// 3. Bitcast the result of the malloc call to the specified type.
1489 static Instruction *CreateMalloc(Instruction *InsertBefore,
1490 Type *IntPtrTy, Type *AllocTy,
1491 Value *AllocSize, Value *ArraySize = nullptr,
1492 Function* MallocF = nullptr,
1493 const Twine &Name = "");
1494 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd,
1495 Type *IntPtrTy, Type *AllocTy,
1496 Value *AllocSize, Value *ArraySize = nullptr,
1497 Function* MallocF = nullptr,
1498 const Twine &Name = "");
1499 static Instruction *CreateMalloc(Instruction *InsertBefore,
1500 Type *IntPtrTy, Type *AllocTy,
1501 Value *AllocSize, Value *ArraySize = nullptr,
1502 ArrayRef<OperandBundleDef> Bundles = None,
1503 Function* MallocF = nullptr,
1504 const Twine &Name = "");
1505 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd,
1506 Type *IntPtrTy, Type *AllocTy,
1507 Value *AllocSize, Value *ArraySize = nullptr,
1508 ArrayRef<OperandBundleDef> Bundles = None,
1509 Function* MallocF = nullptr,
1510 const Twine &Name = "");
1511 /// Generate the IR for a call to the builtin free function.
1512 static Instruction *CreateFree(Value *Source,
1513 Instruction *InsertBefore);
1514 static Instruction *CreateFree(Value *Source,
1515 BasicBlock *InsertAtEnd);
1516 static Instruction *CreateFree(Value *Source,
1517 ArrayRef<OperandBundleDef> Bundles,
1518 Instruction *InsertBefore);
1519 static Instruction *CreateFree(Value *Source,
1520 ArrayRef<OperandBundleDef> Bundles,
1521 BasicBlock *InsertAtEnd);
1522
1523 FunctionType *getFunctionType() const { return FTy; }
1524
1525 void mutateFunctionType(FunctionType *FTy) {
1526 mutateType(FTy->getReturnType());
1527 this->FTy = FTy;
1528 }
1529
1530 // Note that 'musttail' implies 'tail'.
1531 enum TailCallKind { TCK_None = 0, TCK_Tail = 1, TCK_MustTail = 2,
1532 TCK_NoTail = 3 };
1533 TailCallKind getTailCallKind() const {
1534 return TailCallKind(getSubclassDataFromInstruction() & 3);
1535 }
1536
1537 bool isTailCall() const {
1538 unsigned Kind = getSubclassDataFromInstruction() & 3;
1539 return Kind == TCK_Tail || Kind == TCK_MustTail;
1540 }
1541
1542 bool isMustTailCall() const {
1543 return (getSubclassDataFromInstruction() & 3) == TCK_MustTail;
1544 }
1545
1546 bool isNoTailCall() const {
1547 return (getSubclassDataFromInstruction() & 3) == TCK_NoTail;
1548 }
1549
1550 void setTailCall(bool isTC = true) {
1551 setInstructionSubclassData((getSubclassDataFromInstruction() & ~3) |
1552 unsigned(isTC ? TCK_Tail : TCK_None));
1553 }
1554
1555 void setTailCallKind(TailCallKind TCK) {
1556 setInstructionSubclassData((getSubclassDataFromInstruction() & ~3) |
1557 unsigned(TCK));
1558 }
1559
1560 /// Provide fast operand accessors
1561 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
1562
1563 /// Return the number of call arguments.
1564 ///
1565 unsigned getNumArgOperands() const {
1566 return getNumOperands() - getNumTotalBundleOperands() - 1;
1567 }
1568
1569 /// getArgOperand/setArgOperand - Return/set the i-th call argument.
1570 ///
1571 Value *getArgOperand(unsigned i) const {
1572 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1572, __extension__ __PRETTY_FUNCTION__));
1573 return getOperand(i);
1574 }
1575 void setArgOperand(unsigned i, Value *v) {
1576 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1576, __extension__ __PRETTY_FUNCTION__));
1577 setOperand(i, v);
1578 }
1579
1580 /// Return the iterator pointing to the beginning of the argument list.
1581 op_iterator arg_begin() { return op_begin(); }
1582
1583 /// Return the iterator pointing to the end of the argument list.
1584 op_iterator arg_end() {
1585 // [ call args ], [ operand bundles ], callee
1586 return op_end() - getNumTotalBundleOperands() - 1;
1587 }
1588
1589 /// Iteration adapter for range-for loops.
1590 iterator_range<op_iterator> arg_operands() {
1591 return make_range(arg_begin(), arg_end());
1592 }
1593
1594 /// Return the iterator pointing to the beginning of the argument list.
1595 const_op_iterator arg_begin() const { return op_begin(); }
1596
1597 /// Return the iterator pointing to the end of the argument list.
1598 const_op_iterator arg_end() const {
1599 // [ call args ], [ operand bundles ], callee
1600 return op_end() - getNumTotalBundleOperands() - 1;
1601 }
1602
1603 /// Iteration adapter for range-for loops.
1604 iterator_range<const_op_iterator> arg_operands() const {
1605 return make_range(arg_begin(), arg_end());
1606 }
1607
1608 /// Wrappers for getting the \c Use of a call argument.
1609 const Use &getArgOperandUse(unsigned i) const {
1610 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1610, __extension__ __PRETTY_FUNCTION__));
1611 return getOperandUse(i);
1612 }
1613 Use &getArgOperandUse(unsigned i) {
1614 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1614, __extension__ __PRETTY_FUNCTION__));
1615 return getOperandUse(i);
1616 }
1617
1618 /// If one of the arguments has the 'returned' attribute, return its
1619 /// operand value. Otherwise, return nullptr.
1620 Value *getReturnedArgOperand() const;
1621
1622 /// getCallingConv/setCallingConv - Get or set the calling convention of this
1623 /// function call.
1624 CallingConv::ID getCallingConv() const {
1625 return static_cast<CallingConv::ID>(getSubclassDataFromInstruction() >> 2);
1626 }
1627 void setCallingConv(CallingConv::ID CC) {
1628 auto ID = static_cast<unsigned>(CC);
1629 assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention")(static_cast <bool> (!(ID & ~CallingConv::MaxID) &&
"Unsupported calling convention") ? void (0) : __assert_fail
("!(ID & ~CallingConv::MaxID) && \"Unsupported calling convention\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1629, __extension__ __PRETTY_FUNCTION__));
1630 setInstructionSubclassData((getSubclassDataFromInstruction() & 3) |
1631 (ID << 2));
1632 }
1633
1634 /// Return the parameter attributes for this call.
1635 ///
1636 AttributeList getAttributes() const { return Attrs; }
1637
1638 /// Set the parameter attributes for this call.
1639 ///
1640 void setAttributes(AttributeList A) { Attrs = A; }
1641
1642 /// adds the attribute to the list of attributes.
1643 void addAttribute(unsigned i, Attribute::AttrKind Kind);
1644
1645 /// adds the attribute to the list of attributes.
1646 void addAttribute(unsigned i, Attribute Attr);
1647
1648 /// Adds the attribute to the indicated argument
1649 void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind);
1650
1651 /// Adds the attribute to the indicated argument
1652 void addParamAttr(unsigned ArgNo, Attribute Attr);
1653
1654 /// removes the attribute from the list of attributes.
1655 void removeAttribute(unsigned i, Attribute::AttrKind Kind);
1656
1657 /// removes the attribute from the list of attributes.
1658 void removeAttribute(unsigned i, StringRef Kind);
1659
1660 /// Removes the attribute from the given argument
1661 void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind);
1662
1663 /// Removes the attribute from the given argument
1664 void removeParamAttr(unsigned ArgNo, StringRef Kind);
1665
1666 /// adds the dereferenceable attribute to the list of attributes.
1667 void addDereferenceableAttr(unsigned i, uint64_t Bytes);
1668
1669 /// adds the dereferenceable_or_null attribute to the list of
1670 /// attributes.
1671 void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes);
1672
1673 /// Determine whether this call has the given attribute.
1674 bool hasFnAttr(Attribute::AttrKind Kind) const {
1675 assert(Kind != Attribute::NoBuiltin &&(static_cast <bool> (Kind != Attribute::NoBuiltin &&
"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? void (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1676, __extension__ __PRETTY_FUNCTION__))
1676 "Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin")(static_cast <bool> (Kind != Attribute::NoBuiltin &&
"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? void (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1676, __extension__ __PRETTY_FUNCTION__));
1677 return hasFnAttrImpl(Kind);
1678 }
1679
1680 /// Determine whether this call has the given attribute.
1681 bool hasFnAttr(StringRef Kind) const {
1682 return hasFnAttrImpl(Kind);
1683 }
1684
1685 /// Determine whether the return value has the given attribute.
1686 bool hasRetAttr(Attribute::AttrKind Kind) const;
1687
1688 /// Determine whether the argument or parameter has the given attribute.
1689 bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
1690
1691 /// Get the attribute of a given kind at a position.
1692 Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const {
1693 return getAttributes().getAttribute(i, Kind);
1694 }
1695
1696 /// Get the attribute of a given kind at a position.
1697 Attribute getAttribute(unsigned i, StringRef Kind) const {
1698 return getAttributes().getAttribute(i, Kind);
1699 }
1700
1701 /// Get the attribute of a given kind from a given arg
1702 Attribute getParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
1703 assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
"Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1703, __extension__ __PRETTY_FUNCTION__));
1704 return getAttributes().getParamAttr(ArgNo, Kind);
1705 }
1706
1707 /// Get the attribute of a given kind from a given arg
1708 Attribute getParamAttr(unsigned ArgNo, StringRef Kind) const {
1709 assert(ArgNo < getNumArgOperands() && "Out of bounds")(static_cast <bool> (ArgNo < getNumArgOperands() &&
"Out of bounds") ? void (0) : __assert_fail ("ArgNo < getNumArgOperands() && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1709, __extension__ __PRETTY_FUNCTION__));
1710 return getAttributes().getParamAttr(ArgNo, Kind);
1711 }
1712
1713 /// Return true if the data operand at index \p i has the attribute \p
1714 /// A.
1715 ///
1716 /// Data operands include call arguments and values used in operand bundles,
1717 /// but does not include the callee operand. This routine dispatches to the
1718 /// underlying AttributeList or the OperandBundleUser as appropriate.
1719 ///
1720 /// The index \p i is interpreted as
1721 ///
1722 /// \p i == Attribute::ReturnIndex -> the return value
1723 /// \p i in [1, arg_size + 1) -> argument number (\p i - 1)
1724 /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index
1725 /// (\p i - 1) in the operand list.
1726 bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const;
1727
1728 /// Extract the alignment of the return value.
1729 unsigned getRetAlignment() const { return Attrs.getRetAlignment(); }
1730
1731 /// Extract the alignment for a call or parameter (0=unknown).
1732 unsigned getParamAlignment(unsigned ArgNo) const {
1733 return Attrs.getParamAlignment(ArgNo);
1734 }
1735
1736 /// Extract the number of dereferenceable bytes for a call or
1737 /// parameter (0=unknown).
1738 uint64_t getDereferenceableBytes(unsigned i) const {
1739 return Attrs.getDereferenceableBytes(i);
1740 }
1741
1742 /// Extract the number of dereferenceable_or_null bytes for a call or
1743 /// parameter (0=unknown).
1744 uint64_t getDereferenceableOrNullBytes(unsigned i) const {
1745 return Attrs.getDereferenceableOrNullBytes(i);
1746 }
1747
1748 /// @brief Determine if the return value is marked with NoAlias attribute.
1749 bool returnDoesNotAlias() const {
1750 return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
1751 }
1752
1753 /// Return true if the call should not be treated as a call to a
1754 /// builtin.
1755 bool isNoBuiltin() const {
1756 return hasFnAttrImpl(Attribute::NoBuiltin) &&
1757 !hasFnAttrImpl(Attribute::Builtin);
1758 }
1759
1760 /// Determine if the call requires strict floating point semantics.
1761 bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); }
1762
1763 /// Return true if the call should not be inlined.
1764 bool isNoInline() const { return hasFnAttr(Attribute::NoInline); }
1765 void setIsNoInline() {
1766 addAttribute(AttributeList::FunctionIndex, Attribute::NoInline);
1767 }
1768
1769 /// Return true if the call can return twice
1770 bool canReturnTwice() const {
1771 return hasFnAttr(Attribute::ReturnsTwice);
1772 }
1773 void setCanReturnTwice() {
1774 addAttribute(AttributeList::FunctionIndex, Attribute::ReturnsTwice);
1775 }
1776
1777 /// Determine if the call does not access memory.
1778 bool doesNotAccessMemory() const {
1779 return hasFnAttr(Attribute::ReadNone);
1780 }
1781 void setDoesNotAccessMemory() {
1782 addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
1783 }
1784
1785 /// Determine if the call does not access or only reads memory.
1786 bool onlyReadsMemory() const {
1787 return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly);
1788 }
1789 void setOnlyReadsMemory() {
1790 addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly);
1791 }
1792
1793 /// Determine if the call does not access or only writes memory.
1794 bool doesNotReadMemory() const {
1795 return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly);
1796 }
1797 void setDoesNotReadMemory() {
1798 addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly);
1799 }
1800
1801 /// @brief Determine if the call can access memmory only using pointers based
1802 /// on its arguments.
1803 bool onlyAccessesArgMemory() const {
1804 return hasFnAttr(Attribute::ArgMemOnly);
1805 }
1806 void setOnlyAccessesArgMemory() {
1807 addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly);
1808 }
1809
1810 /// @brief Determine if the function may only access memory that is
1811 /// inaccessible from the IR.
1812 bool onlyAccessesInaccessibleMemory() const {
1813 return hasFnAttr(Attribute::InaccessibleMemOnly);
1814 }
1815 void setOnlyAccessesInaccessibleMemory() {
1816 addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly);
1817 }
1818
1819 /// @brief Determine if the function may only access memory that is
1820 /// either inaccessible from the IR or pointed to by its arguments.
1821 bool onlyAccessesInaccessibleMemOrArgMem() const {
1822 return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
1823 }
1824 void setOnlyAccessesInaccessibleMemOrArgMem() {
1825 addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOrArgMemOnly);
1826 }
1827
1828 /// Determine if the call cannot return.
1829 bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
1830 void setDoesNotReturn() {
1831 addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn);
1832 }
1833
1834 /// Determine if the call cannot unwind.
1835 bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); }
1836 void setDoesNotThrow() {
1837 addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
1838 }
1839
1840 /// Determine if the call cannot be duplicated.
1841 bool cannotDuplicate() const {return hasFnAttr(Attribute::NoDuplicate); }
1842 void setCannotDuplicate() {
1843 addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate);
1844 }
1845
1846 /// Determine if the call is convergent
1847 bool isConvergent() const { return hasFnAttr(Attribute::Convergent); }
1848 void setConvergent() {
1849 addAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
1850 }
1851 void setNotConvergent() {
1852 removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
1853 }
1854
1855 /// Determine if the call returns a structure through first
1856 /// pointer argument.
1857 bool hasStructRetAttr() const {
1858 if (getNumArgOperands() == 0)
1859 return false;
1860
1861 // Be friendly and also check the callee.
1862 return paramHasAttr(0, Attribute::StructRet);
1863 }
1864
1865 /// Determine if any call argument is an aggregate passed by value.
1866 bool hasByValArgument() const {
1867 return Attrs.hasAttrSomewhere(Attribute::ByVal);
1868 }
1869
1870 /// Return the function called, or null if this is an
1871 /// indirect function invocation.
1872 ///
1873 Function *getCalledFunction() const {
1874 return dyn_cast<Function>(Op<-1>());
1875 }
1876
1877 /// Get a pointer to the function that is invoked by this
1878 /// instruction.
1879 const Value *getCalledValue() const { return Op<-1>(); }
1880 Value *getCalledValue() { return Op<-1>(); }
1881
1882 /// Set the function called.
1883 void setCalledFunction(Value* Fn) {
1884 setCalledFunction(
1885 cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType()),
1886 Fn);
1887 }
1888 void setCalledFunction(FunctionType *FTy, Value *Fn) {
1889 this->FTy = FTy;
1890 assert(FTy == cast<FunctionType>((static_cast <bool> (FTy == cast<FunctionType>( cast
<PointerType>(Fn->getType())->getElementType())) ?
void (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1891, __extension__ __PRETTY_FUNCTION__))
1891 cast<PointerType>(Fn->getType())->getElementType()))(static_cast <bool> (FTy == cast<FunctionType>( cast
<PointerType>(Fn->getType())->getElementType())) ?
void (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1891, __extension__ __PRETTY_FUNCTION__));
1892 Op<-1>() = Fn;
1893 }
1894
1895 /// Check if this call is an inline asm statement.
1896 bool isInlineAsm() const {
1897 return isa<InlineAsm>(Op<-1>());
1898 }
1899
1900 // Methods for support type inquiry through isa, cast, and dyn_cast:
1901 static bool classof(const Instruction *I) {
1902 return I->getOpcode() == Instruction::Call;
20
←
Calling 'Instruction::getOpcode'
→
23
←
Returning from 'Instruction::getOpcode'
→
24
←
Assuming the condition is true
→
1903 }
1904 static bool classof(const Value *V) {
1905 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1906 }
1907
1908private:
1909 template <typename AttrKind> bool hasFnAttrImpl(AttrKind Kind) const {
1910 if (Attrs.hasAttribute(AttributeList::FunctionIndex, Kind))
1911 return true;
1912
1913 // Operand bundles override attributes on the called function, but don't
1914 // override attributes directly present on the call instruction.
1915 if (isFnAttrDisallowedByOpBundle(Kind))
1916 return false;
1917
1918 if (const Function *F = getCalledFunction())
1919 return F->getAttributes().hasAttribute(AttributeList::FunctionIndex,
1920 Kind);
1921 return false;
1922 }
1923
1924 // Shadow Instruction::setInstructionSubclassData with a private forwarding
1925 // method so that subclasses cannot accidentally use it.
1926 void setInstructionSubclassData(unsigned short D) {
1927 Instruction::setInstructionSubclassData(D);
1928 }
1929};
1930
1931template <>
1932struct OperandTraits<CallInst> : public VariadicOperandTraits<CallInst, 1> {
1933};
1934
1935CallInst::CallInst(Value *Func, ArrayRef<Value *> Args,
1936 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1937 BasicBlock *InsertAtEnd)
1938 : Instruction(
1939 cast<FunctionType>(cast<PointerType>(Func->getType())
1940 ->getElementType())->getReturnType(),
1941 Instruction::Call, OperandTraits<CallInst>::op_end(this) -
1942 (Args.size() + CountBundleInputs(Bundles) + 1),
1943 unsigned(Args.size() + CountBundleInputs(Bundles) + 1), InsertAtEnd) {
1944 init(Func, Args, Bundles, NameStr);
1945}
1946
1947CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1948 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1949 Instruction *InsertBefore)
1950 : Instruction(Ty->getReturnType(), Instruction::Call,
1951 OperandTraits<CallInst>::op_end(this) -
1952 (Args.size() + CountBundleInputs(Bundles) + 1),
1953 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1954 InsertBefore) {
1955 init(Ty, Func, Args, Bundles, NameStr);
1956}
1957
1958// Note: if you get compile errors about private methods then
1959// please update your code to use the high-level operand
1960// interfaces. See line 943 above.
1961DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CallInst, Value)CallInst::op_iterator CallInst::op_begin() { return OperandTraits
<CallInst>::op_begin(this); } CallInst::const_op_iterator
CallInst::op_begin() const { return OperandTraits<CallInst
>::op_begin(const_cast<CallInst*>(this)); } CallInst
::op_iterator CallInst::op_end() { return OperandTraits<CallInst
>::op_end(this); } CallInst::const_op_iterator CallInst::op_end
() const { return OperandTraits<CallInst>::op_end(const_cast
<CallInst*>(this)); } Value *CallInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<CallInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CallInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1961, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<CallInst>::op_begin(const_cast
<CallInst*>(this))[i_nocapture].get()); } void CallInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<CallInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<CallInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1961, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
CallInst>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
CallInst::getNumOperands() const { return OperandTraits<CallInst
>::operands(this); } template <int Idx_nocapture> Use
&CallInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
CallInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
1962
1963//===----------------------------------------------------------------------===//
1964// SelectInst Class
1965//===----------------------------------------------------------------------===//
1966
1967/// This class represents the LLVM 'select' instruction.
1968///
1969class SelectInst : public Instruction {
1970 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1971 Instruction *InsertBefore)
1972 : Instruction(S1->getType(), Instruction::Select,
1973 &Op<0>(), 3, InsertBefore) {
1974 init(C, S1, S2);
1975 setName(NameStr);
1976 }
1977
1978 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1979 BasicBlock *InsertAtEnd)
1980 : Instruction(S1->getType(), Instruction::Select,
1981 &Op<0>(), 3, InsertAtEnd) {
1982 init(C, S1, S2);
1983 setName(NameStr);
1984 }
1985
1986 void init(Value *C, Value *S1, Value *S2) {
1987 assert(!areInvalidOperands(C, S1, S2) && "Invalid operands for select")(static_cast <bool> (!areInvalidOperands(C, S1, S2) &&
"Invalid operands for select") ? void (0) : __assert_fail ("!areInvalidOperands(C, S1, S2) && \"Invalid operands for select\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 1987, __extension__ __PRETTY_FUNCTION__));
1988 Op<0>() = C;
1989 Op<1>() = S1;
1990 Op<2>() = S2;
1991 }
1992
1993protected:
1994 // Note: Instruction needs to be a friend here to call cloneImpl.
1995 friend class Instruction;
1996
1997 SelectInst *cloneImpl() const;
1998
1999public:
2000 static SelectInst *Create(Value *C, Value *S1, Value *S2,
2001 const Twine &NameStr = "",
2002 Instruction *InsertBefore = nullptr,
2003 Instruction *MDFrom = nullptr) {
2004 SelectInst *Sel = new(3) SelectInst(C, S1, S2, NameStr, InsertBefore);
2005 if (MDFrom)
2006 Sel->copyMetadata(*MDFrom);
2007 return Sel;
2008 }
2009
2010 static SelectInst *Create(Value *C, Value *S1, Value *S2,
2011 const Twine &NameStr,
2012 BasicBlock *InsertAtEnd) {
2013 return new(3) SelectInst(C, S1, S2, NameStr, InsertAtEnd);
2014 }
2015
2016 const Value *getCondition() const { return Op<0>(); }
2017 const Value *getTrueValue() const { return Op<1>(); }
2018 const Value *getFalseValue() const { return Op<2>(); }
2019 Value *getCondition() { return Op<0>(); }
2020 Value *getTrueValue() { return Op<1>(); }
2021 Value *getFalseValue() { return Op<2>(); }
2022
2023 void setCondition(Value *V) { Op<0>() = V; }
2024 void setTrueValue(Value *V) { Op<1>() = V; }
2025 void setFalseValue(Value *V) { Op<2>() = V; }
2026
2027 /// Return a string if the specified operands are invalid
2028 /// for a select operation, otherwise return null.
2029 static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
2030
2031 /// Transparently provide more efficient getOperand methods.
2032 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2033
2034 OtherOps getOpcode() const {
2035 return static_cast<OtherOps>(Instruction::getOpcode());
2036 }
2037
2038 // Methods for support type inquiry through isa, cast, and dyn_cast:
2039 static bool classof(const Instruction *I) {
2040 return I->getOpcode() == Instruction::Select;
2041 }
2042 static bool classof(const Value *V) {
2043 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2044 }
2045};
2046
2047template <>
2048struct OperandTraits<SelectInst> : public FixedNumOperandTraits<SelectInst, 3> {
2049};
2050
2051DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)SelectInst::op_iterator SelectInst::op_begin() { return OperandTraits
<SelectInst>::op_begin(this); } SelectInst::const_op_iterator
SelectInst::op_begin() const { return OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this)); } SelectInst
::op_iterator SelectInst::op_end() { return OperandTraits<
SelectInst>::op_end(this); } SelectInst::const_op_iterator
SelectInst::op_end() const { return OperandTraits<SelectInst
>::op_end(const_cast<SelectInst*>(this)); } Value *SelectInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SelectInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2051, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<SelectInst>::op_begin(const_cast
<SelectInst*>(this))[i_nocapture].get()); } void SelectInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<SelectInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2051, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
SelectInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned SelectInst::getNumOperands() const { return OperandTraits
<SelectInst>::operands(this); } template <int Idx_nocapture
> Use &SelectInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
SelectInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
2052
2053//===----------------------------------------------------------------------===//
2054// VAArgInst Class
2055//===----------------------------------------------------------------------===//
2056
2057/// This class represents the va_arg llvm instruction, which returns
2058/// an argument of the specified type given a va_list and increments that list
2059///
2060class VAArgInst : public UnaryInstruction {
2061protected:
2062 // Note: Instruction needs to be a friend here to call cloneImpl.
2063 friend class Instruction;
2064
2065 VAArgInst *cloneImpl() const;
2066
2067public:
2068 VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
2069 Instruction *InsertBefore = nullptr)
2070 : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
2071 setName(NameStr);
2072 }
2073
2074 VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
2075 BasicBlock *InsertAtEnd)
2076 : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
2077 setName(NameStr);
2078 }
2079
2080 Value *getPointerOperand() { return getOperand(0); }
2081 const Value *getPointerOperand() const { return getOperand(0); }
2082 static unsigned getPointerOperandIndex() { return 0U; }
2083
2084 // Methods for support type inquiry through isa, cast, and dyn_cast:
2085 static bool classof(const Instruction *I) {
2086 return I->getOpcode() == VAArg;
2087 }
2088 static bool classof(const Value *V) {
2089 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2090 }
2091};
2092
2093//===----------------------------------------------------------------------===//
2094// ExtractElementInst Class
2095//===----------------------------------------------------------------------===//
2096
2097/// This instruction extracts a single (scalar)
2098/// element from a VectorType value
2099///
2100class ExtractElementInst : public Instruction {
2101 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
2102 Instruction *InsertBefore = nullptr);
2103 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
2104 BasicBlock *InsertAtEnd);
2105
2106protected:
2107 // Note: Instruction needs to be a friend here to call cloneImpl.
2108 friend class Instruction;
2109
2110 ExtractElementInst *cloneImpl() const;
2111
2112public:
2113 static ExtractElementInst *Create(Value *Vec, Value *Idx,
2114 const Twine &NameStr = "",
2115 Instruction *InsertBefore = nullptr) {
2116 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertBefore);
2117 }
2118
2119 static ExtractElementInst *Create(Value *Vec, Value *Idx,
2120 const Twine &NameStr,
2121 BasicBlock *InsertAtEnd) {
2122 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertAtEnd);
2123 }
2124
2125 /// Return true if an extractelement instruction can be
2126 /// formed with the specified operands.
2127 static bool isValidOperands(const Value *Vec, const Value *Idx);
2128
2129 Value *getVectorOperand() { return Op<0>(); }
2130 Value *getIndexOperand() { return Op<1>(); }
2131 const Value *getVectorOperand() const { return Op<0>(); }
2132 const Value *getIndexOperand() const { return Op<1>(); }
2133
2134 VectorType *getVectorOperandType() const {
2135 return cast<VectorType>(getVectorOperand()->getType());
2136 }
2137
2138 /// Transparently provide more efficient getOperand methods.
2139 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2140
2141 // Methods for support type inquiry through isa, cast, and dyn_cast:
2142 static bool classof(const Instruction *I) {
2143 return I->getOpcode() == Instruction::ExtractElement;
2144 }
2145 static bool classof(const Value *V) {
2146 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2147 }
2148};
2149
2150template <>
2151struct OperandTraits<ExtractElementInst> :
2152 public FixedNumOperandTraits<ExtractElementInst, 2> {
2153};
2154
2155DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)ExtractElementInst::op_iterator ExtractElementInst::op_begin(
) { return OperandTraits<ExtractElementInst>::op_begin(
this); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_begin() const { return OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this)); }
ExtractElementInst::op_iterator ExtractElementInst::op_end()
{ return OperandTraits<ExtractElementInst>::op_end(this
); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_end() const { return OperandTraits<ExtractElementInst
>::op_end(const_cast<ExtractElementInst*>(this)); } Value
*ExtractElementInst::getOperand(unsigned i_nocapture) const {
(static_cast <bool> (i_nocapture < OperandTraits<
ExtractElementInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2155, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ExtractElementInst>::op_begin
(const_cast<ExtractElementInst*>(this))[i_nocapture].get
()); } void ExtractElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ExtractElementInst>::operands(this)
&& "setOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2155, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ExtractElementInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned ExtractElementInst::getNumOperands() const { return
OperandTraits<ExtractElementInst>::operands(this); } template
<int Idx_nocapture> Use &ExtractElementInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ExtractElementInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2156
2157//===----------------------------------------------------------------------===//
2158// InsertElementInst Class
2159//===----------------------------------------------------------------------===//
2160
2161/// This instruction inserts a single (scalar)
2162/// element into a VectorType value
2163///
2164class InsertElementInst : public Instruction {
2165 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
2166 const Twine &NameStr = "",
2167 Instruction *InsertBefore = nullptr);
2168 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr,
2169 BasicBlock *InsertAtEnd);
2170
2171protected:
2172 // Note: Instruction needs to be a friend here to call cloneImpl.
2173 friend class Instruction;
2174
2175 InsertElementInst *cloneImpl() const;
2176
2177public:
2178 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
2179 const Twine &NameStr = "",
2180 Instruction *InsertBefore = nullptr) {
2181 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertBefore);
2182 }
2183
2184 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
2185 const Twine &NameStr,
2186 BasicBlock *InsertAtEnd) {
2187 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertAtEnd);
2188 }
2189
2190 /// Return true if an insertelement instruction can be
2191 /// formed with the specified operands.
2192 static bool isValidOperands(const Value *Vec, const Value *NewElt,
2193 const Value *Idx);
2194
2195 /// Overload to return most specific vector type.
2196 ///
2197 VectorType *getType() const {
2198 return cast<VectorType>(Instruction::getType());
2199 }
2200
2201 /// Transparently provide more efficient getOperand methods.
2202 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2203
2204 // Methods for support type inquiry through isa, cast, and dyn_cast:
2205 static bool classof(const Instruction *I) {
2206 return I->getOpcode() == Instruction::InsertElement;
2207 }
2208 static bool classof(const Value *V) {
2209 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2210 }
2211};
2212
2213template <>
2214struct OperandTraits<InsertElementInst> :
2215 public FixedNumOperandTraits<InsertElementInst, 3> {
2216};
2217
2218DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)InsertElementInst::op_iterator InsertElementInst::op_begin() {
return OperandTraits<InsertElementInst>::op_begin(this
); } InsertElementInst::const_op_iterator InsertElementInst::
op_begin() const { return OperandTraits<InsertElementInst>
::op_begin(const_cast<InsertElementInst*>(this)); } InsertElementInst
::op_iterator InsertElementInst::op_end() { return OperandTraits
<InsertElementInst>::op_end(this); } InsertElementInst::
const_op_iterator InsertElementInst::op_end() const { return OperandTraits
<InsertElementInst>::op_end(const_cast<InsertElementInst
*>(this)); } Value *InsertElementInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<InsertElementInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2218, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InsertElementInst>::op_begin
(const_cast<InsertElementInst*>(this))[i_nocapture].get
()); } void InsertElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertElementInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2218, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InsertElementInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned InsertElementInst::getNumOperands() const { return
OperandTraits<InsertElementInst>::operands(this); } template
<int Idx_nocapture> Use &InsertElementInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &InsertElementInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2219
2220//===----------------------------------------------------------------------===//
2221// ShuffleVectorInst Class
2222//===----------------------------------------------------------------------===//
2223
2224/// This instruction constructs a fixed permutation of two
2225/// input vectors.
2226///
2227class ShuffleVectorInst : public Instruction {
2228protected:
2229 // Note: Instruction needs to be a friend here to call cloneImpl.
2230 friend class Instruction;
2231
2232 ShuffleVectorInst *cloneImpl() const;
2233
2234public:
2235 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2236 const Twine &NameStr = "",
2237 Instruction *InsertBefor = nullptr);
2238 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2239 const Twine &NameStr, BasicBlock *InsertAtEnd);
2240
2241 // allocate space for exactly three operands
2242 void *operator new(size_t s) {
2243 return User::operator new(s, 3);
2244 }
2245
2246 /// Return true if a shufflevector instruction can be
2247 /// formed with the specified operands.
2248 static bool isValidOperands(const Value *V1, const Value *V2,
2249 const Value *Mask);
2250
2251 /// Overload to return most specific vector type.
2252 ///
2253 VectorType *getType() const {
2254 return cast<VectorType>(Instruction::getType());
2255 }
2256
2257 /// Transparently provide more efficient getOperand methods.
2258 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2259
2260 Constant *getMask() const {
2261 return cast<Constant>(getOperand(2));
2262 }
2263
2264 /// Return the shuffle mask value for the specified element of the mask.
2265 /// Return -1 if the element is undef.
2266 static int getMaskValue(Constant *Mask, unsigned Elt);
2267
2268 /// Return the shuffle mask value of this instruction for the given element
2269 /// index. Return -1 if the element is undef.
2270 int getMaskValue(unsigned Elt) const {
2271 return getMaskValue(getMask(), Elt);
2272 }
2273
2274 /// Convert the input shuffle mask operand to a vector of integers. Undefined
2275 /// elements of the mask are returned as -1.
2276 static void getShuffleMask(Constant *Mask, SmallVectorImpl<int> &Result);
2277
2278 /// Return the mask for this instruction as a vector of integers. Undefined
2279 /// elements of the mask are returned as -1.
2280 void getShuffleMask(SmallVectorImpl<int> &Result) const {
2281 return getShuffleMask(getMask(), Result);
2282 }
2283
2284 SmallVector<int, 16> getShuffleMask() const {
2285 SmallVector<int, 16> Mask;
2286 getShuffleMask(Mask);
2287 return Mask;
2288 }
2289
2290 /// Change values in a shuffle permute mask assuming the two vector operands
2291 /// of length InVecNumElts have swapped position.
2292 static void commuteShuffleMask(MutableArrayRef<int> Mask,
2293 unsigned InVecNumElts) {
2294 for (int &Idx : Mask) {
2295 if (Idx == -1)
2296 continue;
2297 Idx = Idx < (int)InVecNumElts ? Idx + InVecNumElts : Idx - InVecNumElts;
2298 assert(Idx >= 0 && Idx < (int)InVecNumElts * 2 &&(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2299, __extension__ __PRETTY_FUNCTION__))
2299 "shufflevector mask index out of range")(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2299, __extension__ __PRETTY_FUNCTION__));
2300 }
2301 }
2302
2303 // Methods for support type inquiry through isa, cast, and dyn_cast:
2304 static bool classof(const Instruction *I) {
2305 return I->getOpcode() == Instruction::ShuffleVector;
2306 }
2307 static bool classof(const Value *V) {
2308 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2309 }
2310};
2311
2312template <>
2313struct OperandTraits<ShuffleVectorInst> :
2314 public FixedNumOperandTraits<ShuffleVectorInst, 3> {
2315};
2316
2317DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)ShuffleVectorInst::op_iterator ShuffleVectorInst::op_begin() {
return OperandTraits<ShuffleVectorInst>::op_begin(this
); } ShuffleVectorInst::const_op_iterator ShuffleVectorInst::
op_begin() const { return OperandTraits<ShuffleVectorInst>
::op_begin(const_cast<ShuffleVectorInst*>(this)); } ShuffleVectorInst
::op_iterator ShuffleVectorInst::op_end() { return OperandTraits
<ShuffleVectorInst>::op_end(this); } ShuffleVectorInst::
const_op_iterator ShuffleVectorInst::op_end() const { return OperandTraits
<ShuffleVectorInst>::op_end(const_cast<ShuffleVectorInst
*>(this)); } Value *ShuffleVectorInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<ShuffleVectorInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2317, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ShuffleVectorInst>::op_begin
(const_cast<ShuffleVectorInst*>(this))[i_nocapture].get
()); } void ShuffleVectorInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ShuffleVectorInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2317, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ShuffleVectorInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned ShuffleVectorInst::getNumOperands() const { return
OperandTraits<ShuffleVectorInst>::operands(this); } template
<int Idx_nocapture> Use &ShuffleVectorInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ShuffleVectorInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2318
2319//===----------------------------------------------------------------------===//
2320// ExtractValueInst Class
2321//===----------------------------------------------------------------------===//
2322
2323/// This instruction extracts a struct member or array
2324/// element value from an aggregate value.
2325///
2326class ExtractValueInst : public UnaryInstruction {
2327 SmallVector<unsigned, 4> Indices;
2328
2329 ExtractValueInst(const ExtractValueInst &EVI);
2330
2331 /// Constructors - Create a extractvalue instruction with a base aggregate
2332 /// value and a list of indices. The first ctor can optionally insert before
2333 /// an existing instruction, the second appends the new instruction to the
2334 /// specified BasicBlock.
2335 inline ExtractValueInst(Value *Agg,
2336 ArrayRef<unsigned> Idxs,
2337 const Twine &NameStr,
2338 Instruction *InsertBefore);
2339 inline ExtractValueInst(Value *Agg,
2340 ArrayRef<unsigned> Idxs,
2341 const Twine &NameStr, BasicBlock *InsertAtEnd);
2342
2343 void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
2344
2345protected:
2346 // Note: Instruction needs to be a friend here to call cloneImpl.
2347 friend class Instruction;
2348
2349 ExtractValueInst *cloneImpl() const;
2350
2351public:
2352 static ExtractValueInst *Create(Value *Agg,
2353 ArrayRef<unsigned> Idxs,
2354 const Twine &NameStr = "",
2355 Instruction *InsertBefore = nullptr) {
2356 return new
2357 ExtractValueInst(Agg, Idxs, NameStr, InsertBefore);
2358 }
2359
2360 static ExtractValueInst *Create(Value *Agg,
2361 ArrayRef<unsigned> Idxs,
2362 const Twine &NameStr,
2363 BasicBlock *InsertAtEnd) {
2364 return new ExtractValueInst(Agg, Idxs, NameStr, InsertAtEnd);
2365 }
2366
2367 /// Returns the type of the element that would be extracted
2368 /// with an extractvalue instruction with the specified parameters.
2369 ///
2370 /// Null is returned if the indices are invalid for the specified type.
2371 static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
2372
2373 using idx_iterator = const unsigned*;
2374
2375 inline idx_iterator idx_begin() const { return Indices.begin(); }
2376 inline idx_iterator idx_end() const { return Indices.end(); }
2377 inline iterator_range<idx_iterator> indices() const {
2378 return make_range(idx_begin(), idx_end());
2379 }
2380
2381 Value *getAggregateOperand() {
2382 return getOperand(0);
2383 }
2384 const Value *getAggregateOperand() const {
2385 return getOperand(0);
2386 }
2387 static unsigned getAggregateOperandIndex() {
2388 return 0U; // get index for modifying correct operand
2389 }
2390
2391 ArrayRef<unsigned> getIndices() const {
2392 return Indices;
2393 }
2394
2395 unsigned getNumIndices() const {
2396 return (unsigned)Indices.size();
2397 }
2398
2399 bool hasIndices() const {
2400 return true;
2401 }
2402
2403 // Methods for support type inquiry through isa, cast, and dyn_cast:
2404 static bool classof(const Instruction *I) {
2405 return I->getOpcode() == Instruction::ExtractValue;
2406 }
2407 static bool classof(const Value *V) {
2408 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2409 }
2410};
2411
2412ExtractValueInst::ExtractValueInst(Value *Agg,
2413 ArrayRef<unsigned> Idxs,
2414 const Twine &NameStr,
2415 Instruction *InsertBefore)
2416 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2417 ExtractValue, Agg, InsertBefore) {
2418 init(Idxs, NameStr);
2419}
2420
2421ExtractValueInst::ExtractValueInst(Value *Agg,
2422 ArrayRef<unsigned> Idxs,
2423 const Twine &NameStr,
2424 BasicBlock *InsertAtEnd)
2425 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2426 ExtractValue, Agg, InsertAtEnd) {
2427 init(Idxs, NameStr);
2428}
2429
2430//===----------------------------------------------------------------------===//
2431// InsertValueInst Class
2432//===----------------------------------------------------------------------===//
2433
2434/// This instruction inserts a struct field of array element
2435/// value into an aggregate value.
2436///
2437class InsertValueInst : public Instruction {
2438 SmallVector<unsigned, 4> Indices;
2439
2440 InsertValueInst(const InsertValueInst &IVI);
2441
2442 /// Constructors - Create a insertvalue instruction with a base aggregate
2443 /// value, a value to insert, and a list of indices. The first ctor can
2444 /// optionally insert before an existing instruction, the second appends
2445 /// the new instruction to the specified BasicBlock.
2446 inline InsertValueInst(Value *Agg, Value *Val,
2447 ArrayRef<unsigned> Idxs,
2448 const Twine &NameStr,
2449 Instruction *InsertBefore);
2450 inline InsertValueInst(Value *Agg, Value *Val,
2451 ArrayRef<unsigned> Idxs,
2452 const Twine &NameStr, BasicBlock *InsertAtEnd);
2453
2454 /// Constructors - These two constructors are convenience methods because one
2455 /// and two index insertvalue instructions are so common.
2456 InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
2457 const Twine &NameStr = "",
2458 Instruction *InsertBefore = nullptr);
2459 InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr,
2460 BasicBlock *InsertAtEnd);
2461
2462 void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2463 const Twine &NameStr);
2464
2465protected:
2466 // Note: Instruction needs to be a friend here to call cloneImpl.
2467 friend class Instruction;
2468
2469 InsertValueInst *cloneImpl() const;
2470
2471public:
2472 // allocate space for exactly two operands
2473 void *operator new(size_t s) {
2474 return User::operator new(s, 2);
2475 }
2476
2477 static InsertValueInst *Create(Value *Agg, Value *Val,
2478 ArrayRef<unsigned> Idxs,
2479 const Twine &NameStr = "",
2480 Instruction *InsertBefore = nullptr) {
2481 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertBefore);
2482 }
2483
2484 static InsertValueInst *Create(Value *Agg, Value *Val,
2485 ArrayRef<unsigned> Idxs,
2486 const Twine &NameStr,
2487 BasicBlock *InsertAtEnd) {
2488 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertAtEnd);
2489 }
2490
2491 /// Transparently provide more efficient getOperand methods.
2492 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2493
2494 using idx_iterator = const unsigned*;
2495
2496 inline idx_iterator idx_begin() const { return Indices.begin(); }
2497 inline idx_iterator idx_end() const { return Indices.end(); }
2498 inline iterator_range<idx_iterator> indices() const {
2499 return make_range(idx_begin(), idx_end());
2500 }
2501
2502 Value *getAggregateOperand() {
2503 return getOperand(0);
2504 }
2505 const Value *getAggregateOperand() const {
2506 return getOperand(0);
2507 }
2508 static unsigned getAggregateOperandIndex() {
2509 return 0U; // get index for modifying correct operand
2510 }
2511
2512 Value *getInsertedValueOperand() {
2513 return getOperand(1);
2514 }
2515 const Value *getInsertedValueOperand() const {
2516 return getOperand(1);
2517 }
2518 static unsigned getInsertedValueOperandIndex() {
2519 return 1U; // get index for modifying correct operand
2520 }
2521
2522 ArrayRef<unsigned> getIndices() const {
2523 return Indices;
2524 }
2525
2526 unsigned getNumIndices() const {
2527 return (unsigned)Indices.size();
2528 }
2529
2530 bool hasIndices() const {
2531 return true;
2532 }
2533
2534 // Methods for support type inquiry through isa, cast, and dyn_cast:
2535 static bool classof(const Instruction *I) {
2536 return I->getOpcode() == Instruction::InsertValue;
2537 }
2538 static bool classof(const Value *V) {
2539 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2540 }
2541};
2542
2543template <>
2544struct OperandTraits<InsertValueInst> :
2545 public FixedNumOperandTraits<InsertValueInst, 2> {
2546};
2547
2548InsertValueInst::InsertValueInst(Value *Agg,
2549 Value *Val,
2550 ArrayRef<unsigned> Idxs,
2551 const Twine &NameStr,
2552 Instruction *InsertBefore)
2553 : Instruction(Agg->getType(), InsertValue,
2554 OperandTraits<InsertValueInst>::op_begin(this),
2555 2, InsertBefore) {
2556 init(Agg, Val, Idxs, NameStr);
2557}
2558
2559InsertValueInst::InsertValueInst(Value *Agg,
2560 Value *Val,
2561 ArrayRef<unsigned> Idxs,
2562 const Twine &NameStr,
2563 BasicBlock *InsertAtEnd)
2564 : Instruction(Agg->getType(), InsertValue,
2565 OperandTraits<InsertValueInst>::op_begin(this),
2566 2, InsertAtEnd) {
2567 init(Agg, Val, Idxs, NameStr);
2568}
2569
2570DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)InsertValueInst::op_iterator InsertValueInst::op_begin() { return
OperandTraits<InsertValueInst>::op_begin(this); } InsertValueInst
::const_op_iterator InsertValueInst::op_begin() const { return
OperandTraits<InsertValueInst>::op_begin(const_cast<
InsertValueInst*>(this)); } InsertValueInst::op_iterator InsertValueInst
::op_end() { return OperandTraits<InsertValueInst>::op_end
(this); } InsertValueInst::const_op_iterator InsertValueInst::
op_end() const { return OperandTraits<InsertValueInst>::
op_end(const_cast<InsertValueInst*>(this)); } Value *InsertValueInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<InsertValueInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2570, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InsertValueInst>::op_begin
(const_cast<InsertValueInst*>(this))[i_nocapture].get()
); } void InsertValueInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<InsertValueInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2570, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InsertValueInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned InsertValueInst::getNumOperands() const { return
OperandTraits<InsertValueInst>::operands(this); } template
<int Idx_nocapture> Use &InsertValueInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &InsertValueInst::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
2571
2572//===----------------------------------------------------------------------===//
2573// PHINode Class
2574//===----------------------------------------------------------------------===//
2575
2576// PHINode - The PHINode class is used to represent the magical mystical PHI
2577// node, that can not exist in nature, but can be synthesized in a computer
2578// scientist's overactive imagination.
2579//
2580class PHINode : public Instruction {
2581 /// The number of operands actually allocated. NumOperands is
2582 /// the number actually in use.
2583 unsigned ReservedSpace;
2584
2585 PHINode(const PHINode &PN);
2586
2587 explicit PHINode(Type *Ty, unsigned NumReservedValues,
2588 const Twine &NameStr = "",
2589 Instruction *InsertBefore = nullptr)
2590 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
2591 ReservedSpace(NumReservedValues) {
2592 setName(NameStr);
2593 allocHungoffUses(ReservedSpace);
2594 }
2595
2596 PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
2597 BasicBlock *InsertAtEnd)
2598 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
2599 ReservedSpace(NumReservedValues) {
2600 setName(NameStr);
2601 allocHungoffUses(ReservedSpace);
2602 }
2603
2604protected:
2605 // Note: Instruction needs to be a friend here to call cloneImpl.
2606 friend class Instruction;
2607
2608 PHINode *cloneImpl() const;
2609
2610 // allocHungoffUses - this is more complicated than the generic
2611 // User::allocHungoffUses, because we have to allocate Uses for the incoming
2612 // values and pointers to the incoming blocks, all in one allocation.
2613 void allocHungoffUses(unsigned N) {
2614 User::allocHungoffUses(N, /* IsPhi */ true);
2615 }
2616
2617public:
2618 /// Constructors - NumReservedValues is a hint for the number of incoming
2619 /// edges that this phi node will have (use 0 if you really have no idea).
2620 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2621 const Twine &NameStr = "",
2622 Instruction *InsertBefore = nullptr) {
2623 return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
2624 }
2625
2626 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2627 const Twine &NameStr, BasicBlock *InsertAtEnd) {
2628 return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
2629 }
2630
2631 /// Provide fast operand accessors
2632 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2633
2634 // Block iterator interface. This provides access to the list of incoming
2635 // basic blocks, which parallels the list of incoming values.
2636
2637 using block_iterator = BasicBlock **;
2638 using const_block_iterator = BasicBlock * const *;
2639
2640 block_iterator block_begin() {
2641 Use::UserRef *ref =
2642 reinterpret_cast<Use::UserRef*>(op_begin() + ReservedSpace);
2643 return reinterpret_cast<block_iterator>(ref + 1);
2644 }
2645
2646 const_block_iterator block_begin() const {
2647 const Use::UserRef *ref =
2648 reinterpret_cast<const Use::UserRef*>(op_begin() + ReservedSpace);
2649 return reinterpret_cast<const_block_iterator>(ref + 1);
2650 }
2651
2652 block_iterator block_end() {
2653 return block_begin() + getNumOperands();
2654 }
2655
2656 const_block_iterator block_end() const {
2657 return block_begin() + getNumOperands();
2658 }
2659
2660 iterator_range<block_iterator> blocks() {
2661 return make_range(block_begin(), block_end());
2662 }
2663
2664 iterator_range<const_block_iterator> blocks() const {
2665 return make_range(block_begin(), block_end());
2666 }
2667
2668 op_range incoming_values() { return operands(); }
2669
2670 const_op_range incoming_values() const { return operands(); }
2671
2672 /// Return the number of incoming edges
2673 ///
2674 unsigned getNumIncomingValues() const { return getNumOperands(); }
2675
2676 /// Return incoming value number x
2677 ///
2678 Value *getIncomingValue(unsigned i) const {
2679 return getOperand(i);
2680 }
2681 void setIncomingValue(unsigned i, Value *V) {
2682 assert(V && "PHI node got a null value!")(static_cast <bool> (V && "PHI node got a null value!"
) ? void (0) : __assert_fail ("V && \"PHI node got a null value!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2682, __extension__ __PRETTY_FUNCTION__));
2683 assert(getType() == V->getType() &&(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2684, __extension__ __PRETTY_FUNCTION__))
2684 "All operands to PHI node must be the same type as the PHI node!")(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2684, __extension__ __PRETTY_FUNCTION__));
2685 setOperand(i, V);
2686 }
2687
2688 static unsigned getOperandNumForIncomingValue(unsigned i) {
2689 return i;
2690 }
2691
2692 static unsigned getIncomingValueNumForOperand(unsigned i) {
2693 return i;
2694 }
2695
2696 /// Return incoming basic block number @p i.
2697 ///
2698 BasicBlock *getIncomingBlock(unsigned i) const {
2699 return block_begin()[i];
2700 }
2701
2702 /// Return incoming basic block corresponding
2703 /// to an operand of the PHI.
2704 ///
2705 BasicBlock *getIncomingBlock(const Use &U) const {
2706 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?")(static_cast <bool> (this == U.getUser() && "Iterator doesn't point to PHI's Uses?"
) ? void (0) : __assert_fail ("this == U.getUser() && \"Iterator doesn't point to PHI's Uses?\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2706, __extension__ __PRETTY_FUNCTION__));
2707 return getIncomingBlock(unsigned(&U - op_begin()));
2708 }
2709
2710 /// Return incoming basic block corresponding
2711 /// to value use iterator.
2712 ///
2713 BasicBlock *getIncomingBlock(Value::const_user_iterator I) const {
2714 return getIncomingBlock(I.getUse());
2715 }
2716
2717 void setIncomingBlock(unsigned i, BasicBlock *BB) {
2718 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2718, __extension__ __PRETTY_FUNCTION__));
2719 block_begin()[i] = BB;
2720 }
2721
2722 /// Add an incoming value to the end of the PHI list
2723 ///
2724 void addIncoming(Value *V, BasicBlock *BB) {
2725 if (getNumOperands() == ReservedSpace)
2726 growOperands(); // Get more space!
2727 // Initialize some new operands.
2728 setNumHungOffUseOperands(getNumOperands() + 1);
2729 setIncomingValue(getNumOperands() - 1, V);
2730 setIncomingBlock(getNumOperands() - 1, BB);
2731 }
2732
2733 /// Remove an incoming value. This is useful if a
2734 /// predecessor basic block is deleted. The value removed is returned.
2735 ///
2736 /// If the last incoming value for a PHI node is removed (and DeletePHIIfEmpty
2737 /// is true), the PHI node is destroyed and any uses of it are replaced with
2738 /// dummy values. The only time there should be zero incoming values to a PHI
2739 /// node is when the block is dead, so this strategy is sound.
2740 ///
2741 Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
2742
2743 Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
2744 int Idx = getBasicBlockIndex(BB);
2745 assert(Idx >= 0 && "Invalid basic block argument to remove!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument to remove!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument to remove!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2745, __extension__ __PRETTY_FUNCTION__));
2746 return removeIncomingValue(Idx, DeletePHIIfEmpty);
2747 }
2748
2749 /// Return the first index of the specified basic
2750 /// block in the value list for this PHI. Returns -1 if no instance.
2751 ///
2752 int getBasicBlockIndex(const BasicBlock *BB) const {
2753 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
2754 if (block_begin()[i] == BB)
2755 return i;
2756 return -1;
2757 }
2758
2759 Value *getIncomingValueForBlock(const BasicBlock *BB) const {
2760 int Idx = getBasicBlockIndex(BB);
2761 assert(Idx >= 0 && "Invalid basic block argument!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2761, __extension__ __PRETTY_FUNCTION__));
2762 return getIncomingValue(Idx);
2763 }
2764
2765 /// If the specified PHI node always merges together the
2766 /// same value, return the value, otherwise return null.
2767 Value *hasConstantValue() const;
2768
2769 /// Whether the specified PHI node always merges
2770 /// together the same value, assuming undefs are equal to a unique
2771 /// non-undef value.
2772 bool hasConstantOrUndefValue() const;
2773
2774 /// Methods for support type inquiry through isa, cast, and dyn_cast:
2775 static bool classof(const Instruction *I) {
2776 return I->getOpcode() == Instruction::PHI;
2777 }
2778 static bool classof(const Value *V) {
2779 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2780 }
2781
2782private:
2783 void growOperands();
2784};
2785
2786template <>
2787struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
2788};
2789
2790DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)PHINode::op_iterator PHINode::op_begin() { return OperandTraits
<PHINode>::op_begin(this); } PHINode::const_op_iterator
PHINode::op_begin() const { return OperandTraits<PHINode>
::op_begin(const_cast<PHINode*>(this)); } PHINode::op_iterator
PHINode::op_end() { return OperandTraits<PHINode>::op_end
(this); } PHINode::const_op_iterator PHINode::op_end() const {
return OperandTraits<PHINode>::op_end(const_cast<PHINode
*>(this)); } Value *PHINode::getOperand(unsigned i_nocapture
) const { (static_cast <bool> (i_nocapture < OperandTraits
<PHINode>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2790, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<PHINode>::op_begin(const_cast
<PHINode*>(this))[i_nocapture].get()); } void PHINode::
setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<PHINode>::
operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2790, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
PHINode>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
PHINode::getNumOperands() const { return OperandTraits<PHINode
>::operands(this); } template <int Idx_nocapture> Use
&PHINode::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
PHINode::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
2791
2792//===----------------------------------------------------------------------===//
2793// LandingPadInst Class
2794//===----------------------------------------------------------------------===//
2795
2796//===---------------------------------------------------------------------------
2797/// The landingpad instruction holds all of the information
2798/// necessary to generate correct exception handling. The landingpad instruction
2799/// cannot be moved from the top of a landing pad block, which itself is
2800/// accessible only from the 'unwind' edge of an invoke. This uses the
2801/// SubclassData field in Value to store whether or not the landingpad is a
2802/// cleanup.
2803///
2804class LandingPadInst : public Instruction {
2805 /// The number of operands actually allocated. NumOperands is
2806 /// the number actually in use.
2807 unsigned ReservedSpace;
2808
2809 LandingPadInst(const LandingPadInst &LP);
2810
2811public:
2812 enum ClauseType { Catch, Filter };
2813
2814private:
2815 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2816 const Twine &NameStr, Instruction *InsertBefore);
2817 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2818 const Twine &NameStr, BasicBlock *InsertAtEnd);
2819
2820 // Allocate space for exactly zero operands.
2821 void *operator new(size_t s) {
2822 return User::operator new(s);
2823 }
2824
2825 void growOperands(unsigned Size);
2826 void init(unsigned NumReservedValues, const Twine &NameStr);
2827
2828protected:
2829 // Note: Instruction needs to be a friend here to call cloneImpl.
2830 friend class Instruction;
2831
2832 LandingPadInst *cloneImpl() const;
2833
2834public:
2835 /// Constructors - NumReservedClauses is a hint for the number of incoming
2836 /// clauses that this landingpad will have (use 0 if you really have no idea).
2837 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2838 const Twine &NameStr = "",
2839 Instruction *InsertBefore = nullptr);
2840 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2841 const Twine &NameStr, BasicBlock *InsertAtEnd);
2842
2843 /// Provide fast operand accessors
2844 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2845
2846 /// Return 'true' if this landingpad instruction is a
2847 /// cleanup. I.e., it should be run when unwinding even if its landing pad
2848 /// doesn't catch the exception.
2849 bool isCleanup() const { return getSubclassDataFromInstruction() & 1; }
2850
2851 /// Indicate that this landingpad instruction is a cleanup.
2852 void setCleanup(bool V) {
2853 setInstructionSubclassData((getSubclassDataFromInstruction() & ~1) |
2854 (V ? 1 : 0));
2855 }
2856
2857 /// Add a catch or filter clause to the landing pad.
2858 void addClause(Constant *ClauseVal);
2859
2860 /// Get the value of the clause at index Idx. Use isCatch/isFilter to
2861 /// determine what type of clause this is.
2862 Constant *getClause(unsigned Idx) const {
2863 return cast<Constant>(getOperandList()[Idx]);
2864 }
2865
2866 /// Return 'true' if the clause and index Idx is a catch clause.
2867 bool isCatch(unsigned Idx) const {
2868 return !isa<ArrayType>(getOperandList()[Idx]->getType());
2869 }
2870
2871 /// Return 'true' if the clause and index Idx is a filter clause.
2872 bool isFilter(unsigned Idx) const {
2873 return isa<ArrayType>(getOperandList()[Idx]->getType());
2874 }
2875
2876 /// Get the number of clauses for this landing pad.
2877 unsigned getNumClauses() const { return getNumOperands(); }
2878
2879 /// Grow the size of the operand list to accommodate the new
2880 /// number of clauses.
2881 void reserveClauses(unsigned Size) { growOperands(Size); }
2882
2883 // Methods for support type inquiry through isa, cast, and dyn_cast:
2884 static bool classof(const Instruction *I) {
2885 return I->getOpcode() == Instruction::LandingPad;
2886 }
2887 static bool classof(const Value *V) {
2888 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2889 }
2890};
2891
2892template <>
2893struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<1> {
2894};
2895
2896DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)LandingPadInst::op_iterator LandingPadInst::op_begin() { return
OperandTraits<LandingPadInst>::op_begin(this); } LandingPadInst
::const_op_iterator LandingPadInst::op_begin() const { return
OperandTraits<LandingPadInst>::op_begin(const_cast<
LandingPadInst*>(this)); } LandingPadInst::op_iterator LandingPadInst
::op_end() { return OperandTraits<LandingPadInst>::op_end
(this); } LandingPadInst::const_op_iterator LandingPadInst::op_end
() const { return OperandTraits<LandingPadInst>::op_end
(const_cast<LandingPadInst*>(this)); } Value *LandingPadInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<LandingPadInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<LandingPadInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2896, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<LandingPadInst>::op_begin(
const_cast<LandingPadInst*>(this))[i_nocapture].get());
} void LandingPadInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<LandingPadInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<LandingPadInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2896, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
LandingPadInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned LandingPadInst::getNumOperands() const { return OperandTraits
<LandingPadInst>::operands(this); } template <int Idx_nocapture
> Use &LandingPadInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &LandingPadInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
2897
2898//===----------------------------------------------------------------------===//
2899// ReturnInst Class
2900//===----------------------------------------------------------------------===//
2901
2902//===---------------------------------------------------------------------------
2903/// Return a value (possibly void), from a function. Execution
2904/// does not continue in this function any longer.
2905///
2906class ReturnInst : public TerminatorInst {
2907 ReturnInst(const ReturnInst &RI);
2908
2909private:
2910 // ReturnInst constructors:
2911 // ReturnInst() - 'ret void' instruction
2912 // ReturnInst( null) - 'ret void' instruction
2913 // ReturnInst(Value* X) - 'ret X' instruction
2914 // ReturnInst( null, Inst *I) - 'ret void' instruction, insert before I
2915 // ReturnInst(Value* X, Inst *I) - 'ret X' instruction, insert before I
2916 // ReturnInst( null, BB *B) - 'ret void' instruction, insert @ end of B
2917 // ReturnInst(Value* X, BB *B) - 'ret X' instruction, insert @ end of B
2918 //
2919 // NOTE: If the Value* passed is of type void then the constructor behaves as
2920 // if it was passed NULL.
2921 explicit ReturnInst(LLVMContext &C, Value *retVal = nullptr,
2922 Instruction *InsertBefore = nullptr);
2923 ReturnInst(LLVMContext &C, Value *retVal, BasicBlock *InsertAtEnd);
2924 explicit ReturnInst(LLVMContext &C, BasicBlock *InsertAtEnd);
2925
2926protected:
2927 // Note: Instruction needs to be a friend here to call cloneImpl.
2928 friend class Instruction;
2929
2930 ReturnInst *cloneImpl() const;
2931
2932public:
2933 static ReturnInst* Create(LLVMContext &C, Value *retVal = nullptr,
2934 Instruction *InsertBefore = nullptr) {
2935 return new(!!retVal) ReturnInst(C, retVal, InsertBefore);
2936 }
2937
2938 static ReturnInst* Create(LLVMContext &C, Value *retVal,
2939 BasicBlock *InsertAtEnd) {
2940 return new(!!retVal) ReturnInst(C, retVal, InsertAtEnd);
2941 }
2942
2943 static ReturnInst* Create(LLVMContext &C, BasicBlock *InsertAtEnd) {
2944 return new(0) ReturnInst(C, InsertAtEnd);
2945 }
2946
2947 /// Provide fast operand accessors
2948 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
2949
2950 /// Convenience accessor. Returns null if there is no return value.
2951 Value *getReturnValue() const {
2952 return getNumOperands() != 0 ? getOperand(0) : nullptr;
2953 }
2954
2955 unsigned getNumSuccessors() const { return 0; }
2956
2957 // Methods for support type inquiry through isa, cast, and dyn_cast:
2958 static bool classof(const Instruction *I) {
2959 return (I->getOpcode() == Instruction::Ret);
2960 }
2961 static bool classof(const Value *V) {
2962 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2963 }
2964
2965private:
2966 friend TerminatorInst;
2967
2968 BasicBlock *getSuccessor(unsigned idx) const {
2969 llvm_unreachable("ReturnInst has no successors!")::llvm::llvm_unreachable_internal("ReturnInst has no successors!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2969);
2970 }
2971
2972 void setSuccessor(unsigned idx, BasicBlock *B) {
2973 llvm_unreachable("ReturnInst has no successors!")::llvm::llvm_unreachable_internal("ReturnInst has no successors!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2973);
2974 }
2975};
2976
2977template <>
2978struct OperandTraits<ReturnInst> : public VariadicOperandTraits<ReturnInst> {
2979};
2980
2981DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ReturnInst, Value)ReturnInst::op_iterator ReturnInst::op_begin() { return OperandTraits
<ReturnInst>::op_begin(this); } ReturnInst::const_op_iterator
ReturnInst::op_begin() const { return OperandTraits<ReturnInst
>::op_begin(const_cast<ReturnInst*>(this)); } ReturnInst
::op_iterator ReturnInst::op_end() { return OperandTraits<
ReturnInst>::op_end(this); } ReturnInst::const_op_iterator
ReturnInst::op_end() const { return OperandTraits<ReturnInst
>::op_end(const_cast<ReturnInst*>(this)); } Value *ReturnInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<ReturnInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ReturnInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2981, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ReturnInst>::op_begin(const_cast
<ReturnInst*>(this))[i_nocapture].get()); } void ReturnInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<ReturnInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<ReturnInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 2981, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ReturnInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned ReturnInst::getNumOperands() const { return OperandTraits
<ReturnInst>::operands(this); } template <int Idx_nocapture
> Use &ReturnInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
ReturnInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
2982
2983//===----------------------------------------------------------------------===//
2984// BranchInst Class
2985//===----------------------------------------------------------------------===//
2986
2987//===---------------------------------------------------------------------------
2988/// Conditional or Unconditional Branch instruction.
2989///
2990class BranchInst : public TerminatorInst {
2991 /// Ops list - Branches are strange. The operands are ordered:
2992 /// [Cond, FalseDest,] TrueDest. This makes some accessors faster because
2993 /// they don't have to check for cond/uncond branchness. These are mostly
2994 /// accessed relative from op_end().
2995 BranchInst(const BranchInst &BI);
2996 // BranchInst constructors (where {B, T, F} are blocks, and C is a condition):
2997 // BranchInst(BB *B) - 'br B'
2998 // BranchInst(BB* T, BB *F, Value *C) - 'br C, T, F'
2999 // BranchInst(BB* B, Inst *I) - 'br B' insert before I
3000 // BranchInst(BB* T, BB *F, Value *C, Inst *I) - 'br C, T, F', insert before I
3001 // BranchInst(BB* B, BB *I) - 'br B' insert at end
3002 // BranchInst(BB* T, BB *F, Value *C, BB *I) - 'br C, T, F', insert at end
3003 explicit BranchInst(BasicBlock *IfTrue, Instruction *InsertBefore = nullptr);
3004 BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
3005 Instruction *InsertBefore = nullptr);
3006 BranchInst(BasicBlock *IfTrue, BasicBlock *InsertAtEnd);
3007 BranchInst(BasicBlock *IfTrue, BasicBlock *IfFalse, Value *Cond,
3008 BasicBlock *InsertAtEnd);
3009
3010 void AssertOK();
3011
3012protected:
3013 // Note: Instruction needs to be a friend here to call cloneImpl.
3014 friend class Instruction;
3015
3016 BranchInst *cloneImpl() const;
3017
3018public:
3019 static BranchInst *Create(BasicBlock *IfTrue,
3020 Instruction *InsertBefore = nullptr) {
3021 return new(1) BranchInst(IfTrue, InsertBefore);
3022 }
3023
3024 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *IfFalse,
3025 Value *Cond, Instruction *InsertBefore = nullptr) {
3026 return new(3) BranchInst(IfTrue, IfFalse, Cond, InsertBefore);
3027 }
3028
3029 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd) {
3030 return new(1) BranchInst(IfTrue, InsertAtEnd);
3031 }
3032
3033 static BranchInst *Create(BasicBlock *IfTrue, BasicBlock *IfFalse,
3034 Value *Cond, BasicBlock *InsertAtEnd) {
3035 return new(3) BranchInst(IfTrue, IfFalse, Cond, InsertAtEnd);
3036 }
3037
3038 /// Transparently provide more efficient getOperand methods.
3039 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
3040
3041 bool isUnconditional() const { return getNumOperands() == 1; }
3042 bool isConditional() const { return getNumOperands() == 3; }
3043
3044 Value *getCondition() const {
3045 assert(isConditional() && "Cannot get condition of an uncond branch!")(static_cast <bool> (isConditional() && "Cannot get condition of an uncond branch!"
) ? void (0) : __assert_fail ("isConditional() && \"Cannot get condition of an uncond branch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3045, __extension__ __PRETTY_FUNCTION__));
3046 return Op<-3>();
3047 }
3048
3049 void setCondition(Value *V) {
3050 assert(isConditional() && "Cannot set condition of unconditional branch!")(static_cast <bool> (isConditional() && "Cannot set condition of unconditional branch!"
) ? void (0) : __assert_fail ("isConditional() && \"Cannot set condition of unconditional branch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3050, __extension__ __PRETTY_FUNCTION__));
3051 Op<-3>() = V;
3052 }
3053
3054 unsigned getNumSuccessors() const { return 1+isConditional(); }
3055
3056 BasicBlock *getSuccessor(unsigned i) const {
3057 assert(i < getNumSuccessors() && "Successor # out of range for Branch!")(static_cast <bool> (i < getNumSuccessors() &&
"Successor # out of range for Branch!") ? void (0) : __assert_fail
("i < getNumSuccessors() && \"Successor # out of range for Branch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3057, __extension__ __PRETTY_FUNCTION__));
3058 return cast_or_null<BasicBlock>((&Op<-1>() - i)->get());
3059 }
3060
3061 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
3062 assert(idx < getNumSuccessors() && "Successor # out of range for Branch!")(static_cast <bool> (idx < getNumSuccessors() &&
"Successor # out of range for Branch!") ? void (0) : __assert_fail
("idx < getNumSuccessors() && \"Successor # out of range for Branch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3062, __extension__ __PRETTY_FUNCTION__));
3063 *(&Op<-1>() - idx) = NewSucc;
3064 }
3065
3066 /// Swap the successors of this branch instruction.
3067 ///
3068 /// Swaps the successors of the branch instruction. This also swaps any
3069 /// branch weight metadata associated with the instruction so that it
3070 /// continues to map correctly to each operand.
3071 void swapSuccessors();
3072
3073 // Methods for support type inquiry through isa, cast, and dyn_cast:
3074 static bool classof(const Instruction *I) {
3075 return (I->getOpcode() == Instruction::Br);
3076 }
3077 static bool classof(const Value *V) {
3078 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3079 }
3080};
3081
3082template <>
3083struct OperandTraits<BranchInst> : public VariadicOperandTraits<BranchInst, 1> {
3084};
3085
3086DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BranchInst, Value)BranchInst::op_iterator BranchInst::op_begin() { return OperandTraits
<BranchInst>::op_begin(this); } BranchInst::const_op_iterator
BranchInst::op_begin() const { return OperandTraits<BranchInst
>::op_begin(const_cast<BranchInst*>(this)); } BranchInst
::op_iterator BranchInst::op_end() { return OperandTraits<
BranchInst>::op_end(this); } BranchInst::const_op_iterator
BranchInst::op_end() const { return OperandTraits<BranchInst
>::op_end(const_cast<BranchInst*>(this)); } Value *BranchInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<BranchInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<BranchInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3086, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<BranchInst>::op_begin(const_cast
<BranchInst*>(this))[i_nocapture].get()); } void BranchInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<BranchInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<BranchInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3086, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
BranchInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned BranchInst::getNumOperands() const { return OperandTraits
<BranchInst>::operands(this); } template <int Idx_nocapture
> Use &BranchInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
BranchInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
3087
3088//===----------------------------------------------------------------------===//
3089// SwitchInst Class
3090//===----------------------------------------------------------------------===//
3091
3092//===---------------------------------------------------------------------------
3093/// Multiway switch
3094///
3095class SwitchInst : public TerminatorInst {
3096 unsigned ReservedSpace;
3097
3098 // Operand[0] = Value to switch on
3099 // Operand[1] = Default basic block destination
3100 // Operand[2n ] = Value to match
3101 // Operand[2n+1] = BasicBlock to go to on match
3102 SwitchInst(const SwitchInst &SI);
3103
3104 /// Create a new switch instruction, specifying a value to switch on and a
3105 /// default destination. The number of additional cases can be specified here
3106 /// to make memory allocation more efficient. This constructor can also
3107 /// auto-insert before another instruction.
3108 SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
3109 Instruction *InsertBefore);
3110
3111 /// Create a new switch instruction, specifying a value to switch on and a
3112 /// default destination. The number of additional cases can be specified here
3113 /// to make memory allocation more efficient. This constructor also
3114 /// auto-inserts at the end of the specified BasicBlock.
3115 SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases,
3116 BasicBlock *InsertAtEnd);
3117
3118 // allocate space for exactly zero operands
3119 void *operator new(size_t s) {
3120 return User::operator new(s);
3121 }
3122
3123 void init(Value *Value, BasicBlock *Default, unsigned NumReserved);
3124 void growOperands();
3125
3126protected:
3127 // Note: Instruction needs to be a friend here to call cloneImpl.
3128 friend class Instruction;
3129
3130 SwitchInst *cloneImpl() const;
3131
3132public:
3133 // -2
3134 static const unsigned DefaultPseudoIndex = static_cast<unsigned>(~0L-1);
3135
3136 template <typename CaseHandleT> class CaseIteratorImpl;
3137
3138 /// A handle to a particular switch case. It exposes a convenient interface
3139 /// to both the case value and the successor block.
3140 ///
3141 /// We define this as a template and instantiate it to form both a const and
3142 /// non-const handle.
3143 template <typename SwitchInstT, typename ConstantIntT, typename BasicBlockT>
3144 class CaseHandleImpl {
3145 // Directly befriend both const and non-const iterators.
3146 friend class SwitchInst::CaseIteratorImpl<
3147 CaseHandleImpl<SwitchInstT, ConstantIntT, BasicBlockT>>;
3148
3149 protected:
3150 // Expose the switch type we're parameterized with to the iterator.
3151 using SwitchInstType = SwitchInstT;
3152
3153 SwitchInstT *SI;
3154 ptrdiff_t Index;
3155
3156 CaseHandleImpl() = default;
3157 CaseHandleImpl(SwitchInstT *SI, ptrdiff_t Index) : SI(SI), Index(Index) {}
3158
3159 public:
3160 /// Resolves case value for current case.
3161 ConstantIntT *getCaseValue() const {
3162 assert((unsigned)Index < SI->getNumCases() &&(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3163, __extension__ __PRETTY_FUNCTION__))
3163 "Index out the number of cases.")(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3163, __extension__ __PRETTY_FUNCTION__));
3164 return reinterpret_cast<ConstantIntT *>(SI->getOperand(2 + Index * 2));
3165 }
3166
3167 /// Resolves successor for current case.
3168 BasicBlockT *getCaseSuccessor() const {
3169 assert(((unsigned)Index < SI->getNumCases() ||(static_cast <bool> (((unsigned)Index < SI->getNumCases
() || (unsigned)Index == DefaultPseudoIndex) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index < SI->getNumCases() || (unsigned)Index == DefaultPseudoIndex) && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3171, __extension__ __PRETTY_FUNCTION__))
3170 (unsigned)Index == DefaultPseudoIndex) &&(static_cast <bool> (((unsigned)Index < SI->getNumCases
() || (unsigned)Index == DefaultPseudoIndex) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index < SI->getNumCases() || (unsigned)Index == DefaultPseudoIndex) && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3171, __extension__ __PRETTY_FUNCTION__))
3171 "Index out the number of cases.")(static_cast <bool> (((unsigned)Index < SI->getNumCases
() || (unsigned)Index == DefaultPseudoIndex) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index < SI->getNumCases() || (unsigned)Index == DefaultPseudoIndex) && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3171, __extension__ __PRETTY_FUNCTION__));
3172 return SI->getSuccessor(getSuccessorIndex());
3173 }
3174
3175 /// Returns number of current case.
3176 unsigned getCaseIndex() const { return Index; }
3177
3178 /// Returns TerminatorInst's successor index for current case successor.
3179 unsigned getSuccessorIndex() const {
3180 assert(((unsigned)Index == DefaultPseudoIndex ||(static_cast <bool> (((unsigned)Index == DefaultPseudoIndex
|| (unsigned)Index < SI->getNumCases()) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index == DefaultPseudoIndex || (unsigned)Index < SI->getNumCases()) && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3182, __extension__ __PRETTY_FUNCTION__))
3181 (unsigned)Index < SI->getNumCases()) &&(static_cast <bool> (((unsigned)Index == DefaultPseudoIndex
|| (unsigned)Index < SI->getNumCases()) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index == DefaultPseudoIndex || (unsigned)Index < SI->getNumCases()) && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3182, __extension__ __PRETTY_FUNCTION__))
3182 "Index out the number of cases.")(static_cast <bool> (((unsigned)Index == DefaultPseudoIndex
|| (unsigned)Index < SI->getNumCases()) && "Index out the number of cases."
) ? void (0) : __assert_fail ("((unsigned)Index == DefaultPseudoIndex || (unsigned)Index < SI->getNumCases()) && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3182, __extension__ __PRETTY_FUNCTION__));
3183 return (unsigned)Index != DefaultPseudoIndex ? Index + 1 : 0;
3184 }
3185
3186 bool operator==(const CaseHandleImpl &RHS) const {
3187 assert(SI == RHS.SI && "Incompatible operators.")(static_cast <bool> (SI == RHS.SI && "Incompatible operators."
) ? void (0) : __assert_fail ("SI == RHS.SI && \"Incompatible operators.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3187, __extension__ __PRETTY_FUNCTION__));
3188 return Index == RHS.Index;
3189 }
3190 };
3191
3192 using ConstCaseHandle =
3193 CaseHandleImpl<const SwitchInst, const ConstantInt, const BasicBlock>;
3194
3195 class CaseHandle
3196 : public CaseHandleImpl<SwitchInst, ConstantInt, BasicBlock> {
3197 friend class SwitchInst::CaseIteratorImpl<CaseHandle>;
3198
3199 public:
3200 CaseHandle(SwitchInst *SI, ptrdiff_t Index) : CaseHandleImpl(SI, Index) {}
3201
3202 /// Sets the new value for current case.
3203 void setValue(ConstantInt *V) {
3204 assert((unsigned)Index < SI->getNumCases() &&(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3205, __extension__ __PRETTY_FUNCTION__))
3205 "Index out the number of cases.")(static_cast <bool> ((unsigned)Index < SI->getNumCases
() && "Index out the number of cases.") ? void (0) : __assert_fail
("(unsigned)Index < SI->getNumCases() && \"Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3205, __extension__ __PRETTY_FUNCTION__));
3206 SI->setOperand(2 + Index*2, reinterpret_cast<Value*>(V));
3207 }
3208
3209 /// Sets the new successor for current case.
3210 void setSuccessor(BasicBlock *S) {
3211 SI->setSuccessor(getSuccessorIndex(), S);
3212 }
3213 };
3214
3215 template <typename CaseHandleT>
3216 class CaseIteratorImpl
3217 : public iterator_facade_base<CaseIteratorImpl<CaseHandleT>,
3218 std::random_access_iterator_tag,
3219 CaseHandleT> {
3220 using SwitchInstT = typename CaseHandleT::SwitchInstType;
3221
3222 CaseHandleT Case;
3223
3224 public:
3225 /// Default constructed iterator is in an invalid state until assigned to
3226 /// a case for a particular switch.
3227 CaseIteratorImpl() = default;
3228
3229 /// Initializes case iterator for given SwitchInst and for given
3230 /// case number.
3231 CaseIteratorImpl(SwitchInstT *SI, unsigned CaseNum) : Case(SI, CaseNum) {}
3232
3233 /// Initializes case iterator for given SwitchInst and for given
3234 /// TerminatorInst's successor index.
3235 static CaseIteratorImpl fromSuccessorIndex(SwitchInstT *SI,
3236 unsigned SuccessorIndex) {
3237 assert(SuccessorIndex < SI->getNumSuccessors() &&(static_cast <bool> (SuccessorIndex < SI->getNumSuccessors
() && "Successor index # out of range!") ? void (0) :
__assert_fail ("SuccessorIndex < SI->getNumSuccessors() && \"Successor index # out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3238, __extension__ __PRETTY_FUNCTION__))
3238 "Successor index # out of range!")(static_cast <bool> (SuccessorIndex < SI->getNumSuccessors
() && "Successor index # out of range!") ? void (0) :
__assert_fail ("SuccessorIndex < SI->getNumSuccessors() && \"Successor index # out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3238, __extension__ __PRETTY_FUNCTION__));
3239 return SuccessorIndex != 0 ? CaseIteratorImpl(SI, SuccessorIndex - 1)
3240 : CaseIteratorImpl(SI, DefaultPseudoIndex);
3241 }
3242
3243 /// Support converting to the const variant. This will be a no-op for const
3244 /// variant.
3245 operator CaseIteratorImpl<ConstCaseHandle>() const {
3246 return CaseIteratorImpl<ConstCaseHandle>(Case.SI, Case.Index);
3247 }
3248
3249 CaseIteratorImpl &operator+=(ptrdiff_t N) {
3250 // Check index correctness after addition.
3251 // Note: Index == getNumCases() means end().
3252 assert(Case.Index + N >= 0 &&(static_cast <bool> (Case.Index + N >= 0 && (
unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index + N >= 0 && (unsigned)(Case.Index + N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3254, __extension__ __PRETTY_FUNCTION__))
3253 (unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&(static_cast <bool> (Case.Index + N >= 0 && (
unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index + N >= 0 && (unsigned)(Case.Index + N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3254, __extension__ __PRETTY_FUNCTION__))
3254 "Case.Index out the number of cases.")(static_cast <bool> (Case.Index + N >= 0 && (
unsigned)(Case.Index + N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index + N >= 0 && (unsigned)(Case.Index + N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3254, __extension__ __PRETTY_FUNCTION__));
3255 Case.Index += N;
3256 return *this;
3257 }
3258 CaseIteratorImpl &operator-=(ptrdiff_t N) {
3259 // Check index correctness after subtraction.
3260 // Note: Case.Index == getNumCases() means end().
3261 assert(Case.Index - N >= 0 &&(static_cast <bool> (Case.Index - N >= 0 && (
unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index - N >= 0 && (unsigned)(Case.Index - N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3263, __extension__ __PRETTY_FUNCTION__))
3262 (unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&(static_cast <bool> (Case.Index - N >= 0 && (
unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index - N >= 0 && (unsigned)(Case.Index - N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3263, __extension__ __PRETTY_FUNCTION__))
3263 "Case.Index out the number of cases.")(static_cast <bool> (Case.Index - N >= 0 && (
unsigned)(Case.Index - N) <= Case.SI->getNumCases() &&
"Case.Index out the number of cases.") ? void (0) : __assert_fail
("Case.Index - N >= 0 && (unsigned)(Case.Index - N) <= Case.SI->getNumCases() && \"Case.Index out the number of cases.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3263, __extension__ __PRETTY_FUNCTION__));
3264 Case.Index -= N;
3265 return *this;
3266 }
3267 ptrdiff_t operator-(const CaseIteratorImpl &RHS) const {
3268 assert(Case.SI == RHS.Case.SI && "Incompatible operators.")(static_cast <bool> (Case.SI == RHS.Case.SI && "Incompatible operators."
) ? void (0) : __assert_fail ("Case.SI == RHS.Case.SI && \"Incompatible operators.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3268, __extension__ __PRETTY_FUNCTION__));
3269 return Case.Index - RHS.Case.Index;
3270 }
3271 bool operator==(const CaseIteratorImpl &RHS) const {
3272 return Case == RHS.Case;
3273 }
3274 bool operator<(const CaseIteratorImpl &RHS) const {
3275 assert(Case.SI == RHS.Case.SI && "Incompatible operators.")(static_cast <bool> (Case.SI == RHS.Case.SI && "Incompatible operators."
) ? void (0) : __assert_fail ("Case.SI == RHS.Case.SI && \"Incompatible operators.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3275, __extension__ __PRETTY_FUNCTION__));
3276 return Case.Index < RHS.Case.Index;
3277 }
3278 CaseHandleT &operator*() { return Case; }
3279 const CaseHandleT &operator*() const { return Case; }
3280 };
3281
3282 using CaseIt = CaseIteratorImpl<CaseHandle>;
3283 using ConstCaseIt = CaseIteratorImpl<ConstCaseHandle>;
3284
3285 static SwitchInst *Create(Value *Value, BasicBlock *Default,
3286 unsigned NumCases,
3287 Instruction *InsertBefore = nullptr) {
3288 return new SwitchInst(Value, Default, NumCases, InsertBefore);
3289 }
3290
3291 static SwitchInst *Create(Value *Value, BasicBlock *Default,
3292 unsigned NumCases, BasicBlock *InsertAtEnd) {
3293 return new SwitchInst(Value, Default, NumCases, InsertAtEnd);
3294 }
3295
3296 /// Provide fast operand accessors
3297 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
3298
3299 // Accessor Methods for Switch stmt
3300 Value *getCondition() const { return getOperand(0); }
3301 void setCondition(Value *V) { setOperand(0, V); }
3302
3303 BasicBlock *getDefaultDest() const {
3304 return cast<BasicBlock>(getOperand(1));
3305 }
3306
3307 void setDefaultDest(BasicBlock *DefaultCase) {
3308 setOperand(1, reinterpret_cast<Value*>(DefaultCase));
3309 }
3310
3311 /// Return the number of 'cases' in this switch instruction, excluding the
3312 /// default case.
3313 unsigned getNumCases() const {
3314 return getNumOperands()/2 - 1;
3315 }
3316
3317 /// Returns a read/write iterator that points to the first case in the
3318 /// SwitchInst.
3319 CaseIt case_begin() {
3320 return CaseIt(this, 0);
3321 }
3322
3323 /// Returns a read-only iterator that points to the first case in the
3324 /// SwitchInst.
3325 ConstCaseIt case_begin() const {
3326 return ConstCaseIt(this, 0);
3327 }
3328
3329 /// Returns a read/write iterator that points one past the last in the
3330 /// SwitchInst.
3331 CaseIt case_end() {
3332 return CaseIt(this, getNumCases());
3333 }
3334
3335 /// Returns a read-only iterator that points one past the last in the
3336 /// SwitchInst.
3337 ConstCaseIt case_end() const {
3338 return ConstCaseIt(this, getNumCases());
3339 }
3340
3341 /// Iteration adapter for range-for loops.
3342 iterator_range<CaseIt> cases() {
3343 return make_range(case_begin(), case_end());
3344 }
3345
3346 /// Constant iteration adapter for range-for loops.
3347 iterator_range<ConstCaseIt> cases() const {
3348 return make_range(case_begin(), case_end());
3349 }
3350
3351 /// Returns an iterator that points to the default case.
3352 /// Note: this iterator allows to resolve successor only. Attempt
3353 /// to resolve case value causes an assertion.
3354 /// Also note, that increment and decrement also causes an assertion and
3355 /// makes iterator invalid.
3356 CaseIt case_default() {
3357 return CaseIt(this, DefaultPseudoIndex);
3358 }
3359 ConstCaseIt case_default() const {
3360 return ConstCaseIt(this, DefaultPseudoIndex);
3361 }
3362
3363 /// Search all of the case values for the specified constant. If it is
3364 /// explicitly handled, return the case iterator of it, otherwise return
3365 /// default case iterator to indicate that it is handled by the default
3366 /// handler.
3367 CaseIt findCaseValue(const ConstantInt *C) {
3368 CaseIt I = llvm::find_if(
3369 cases(), [C](CaseHandle &Case) { return Case.getCaseValue() == C; });
3370 if (I != case_end())
3371 return I;
3372
3373 return case_default();
3374 }
3375 ConstCaseIt findCaseValue(const ConstantInt *C) const {
3376 ConstCaseIt I = llvm::find_if(cases(), [C](ConstCaseHandle &Case) {
3377 return Case.getCaseValue() == C;
3378 });
3379 if (I != case_end())
3380 return I;
3381
3382 return case_default();
3383 }
3384
3385 /// Finds the unique case value for a given successor. Returns null if the
3386 /// successor is not found, not unique, or is the default case.
3387 ConstantInt *findCaseDest(BasicBlock *BB) {
3388 if (BB == getDefaultDest())
3389 return nullptr;
3390
3391 ConstantInt *CI = nullptr;
3392 for (auto Case : cases()) {
3393 if (Case.getCaseSuccessor() != BB)
3394 continue;
3395
3396 if (CI)
3397 return nullptr; // Multiple cases lead to BB.
3398
3399 CI = Case.getCaseValue();
3400 }
3401
3402 return CI;
3403 }
3404
3405 /// Add an entry to the switch instruction.
3406 /// Note:
3407 /// This action invalidates case_end(). Old case_end() iterator will
3408 /// point to the added case.
3409 void addCase(ConstantInt *OnVal, BasicBlock *Dest);
3410
3411 /// This method removes the specified case and its successor from the switch
3412 /// instruction. Note that this operation may reorder the remaining cases at
3413 /// index idx and above.
3414 /// Note:
3415 /// This action invalidates iterators for all cases following the one removed,
3416 /// including the case_end() iterator. It returns an iterator for the next
3417 /// case.
3418 CaseIt removeCase(CaseIt I);
3419
3420 unsigned getNumSuccessors() const { return getNumOperands()/2; }
3421 BasicBlock *getSuccessor(unsigned idx) const {
3422 assert(idx < getNumSuccessors() &&"Successor idx out of range for switch!")(static_cast <bool> (idx < getNumSuccessors() &&
"Successor idx out of range for switch!") ? void (0) : __assert_fail
("idx < getNumSuccessors() &&\"Successor idx out of range for switch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3422, __extension__ __PRETTY_FUNCTION__));
3423 return cast<BasicBlock>(getOperand(idx*2+1));
3424 }
3425 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
3426 assert(idx < getNumSuccessors() && "Successor # out of range for switch!")(static_cast <bool> (idx < getNumSuccessors() &&
"Successor # out of range for switch!") ? void (0) : __assert_fail
("idx < getNumSuccessors() && \"Successor # out of range for switch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3426, __extension__ __PRETTY_FUNCTION__));
3427 setOperand(idx * 2 + 1, NewSucc);
3428 }
3429
3430 // Methods for support type inquiry through isa, cast, and dyn_cast:
3431 static bool classof(const Instruction *I) {
3432 return I->getOpcode() == Instruction::Switch;
3433 }
3434 static bool classof(const Value *V) {
3435 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3436 }
3437};
3438
3439template <>
3440struct OperandTraits<SwitchInst> : public HungoffOperandTraits<2> {
3441};
3442
3443DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SwitchInst, Value)SwitchInst::op_iterator SwitchInst::op_begin() { return OperandTraits
<SwitchInst>::op_begin(this); } SwitchInst::const_op_iterator
SwitchInst::op_begin() const { return OperandTraits<SwitchInst
>::op_begin(const_cast<SwitchInst*>(this)); } SwitchInst
::op_iterator SwitchInst::op_end() { return OperandTraits<
SwitchInst>::op_end(this); } SwitchInst::const_op_iterator
SwitchInst::op_end() const { return OperandTraits<SwitchInst
>::op_end(const_cast<SwitchInst*>(this)); } Value *SwitchInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SwitchInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SwitchInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3443, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<SwitchInst>::op_begin(const_cast
<SwitchInst*>(this))[i_nocapture].get()); } void SwitchInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<SwitchInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<SwitchInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3443, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
SwitchInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned SwitchInst::getNumOperands() const { return OperandTraits
<SwitchInst>::operands(this); } template <int Idx_nocapture
> Use &SwitchInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
SwitchInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
3444
3445//===----------------------------------------------------------------------===//
3446// IndirectBrInst Class
3447//===----------------------------------------------------------------------===//
3448
3449//===---------------------------------------------------------------------------
3450/// Indirect Branch Instruction.
3451///
3452class IndirectBrInst : public TerminatorInst {
3453 unsigned ReservedSpace;
3454
3455 // Operand[0] = Address to jump to
3456 // Operand[n+1] = n-th destination
3457 IndirectBrInst(const IndirectBrInst &IBI);
3458
3459 /// Create a new indirectbr instruction, specifying an
3460 /// Address to jump to. The number of expected destinations can be specified
3461 /// here to make memory allocation more efficient. This constructor can also
3462 /// autoinsert before another instruction.
3463 IndirectBrInst(Value *Address, unsigned NumDests, Instruction *InsertBefore);
3464
3465 /// Create a new indirectbr instruction, specifying an
3466 /// Address to jump to. The number of expected destinations can be specified
3467 /// here to make memory allocation more efficient. This constructor also
3468 /// autoinserts at the end of the specified BasicBlock.
3469 IndirectBrInst(Value *Address, unsigned NumDests, BasicBlock *InsertAtEnd);
3470
3471 // allocate space for exactly zero operands
3472 void *operator new(size_t s) {
3473 return User::operator new(s);
3474 }
3475
3476 void init(Value *Address, unsigned NumDests);
3477 void growOperands();
3478
3479protected:
3480 // Note: Instruction needs to be a friend here to call cloneImpl.
3481 friend class Instruction;
3482
3483 IndirectBrInst *cloneImpl() const;
3484
3485public:
3486 static IndirectBrInst *Create(Value *Address, unsigned NumDests,
3487 Instruction *InsertBefore = nullptr) {
3488 return new IndirectBrInst(Address, NumDests, InsertBefore);
3489 }
3490
3491 static IndirectBrInst *Create(Value *Address, unsigned NumDests,
3492 BasicBlock *InsertAtEnd) {
3493 return new IndirectBrInst(Address, NumDests, InsertAtEnd);
3494 }
3495
3496 /// Provide fast operand accessors.
3497 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
3498
3499 // Accessor Methods for IndirectBrInst instruction.
3500 Value *getAddress() { return getOperand(0); }
3501 const Value *getAddress() const { return getOperand(0); }
3502 void setAddress(Value *V) { setOperand(0, V); }
3503
3504 /// return the number of possible destinations in this
3505 /// indirectbr instruction.
3506 unsigned getNumDestinations() const { return getNumOperands()-1; }
3507
3508 /// Return the specified destination.
3509 BasicBlock *getDestination(unsigned i) { return getSuccessor(i); }
3510 const BasicBlock *getDestination(unsigned i) const { return getSuccessor(i); }
3511
3512 /// Add a destination.
3513 ///
3514 void addDestination(BasicBlock *Dest);
3515
3516 /// This method removes the specified successor from the
3517 /// indirectbr instruction.
3518 void removeDestination(unsigned i);
3519
3520 unsigned getNumSuccessors() const { return getNumOperands()-1; }
3521 BasicBlock *getSuccessor(unsigned i) const {
3522 return cast<BasicBlock>(getOperand(i+1));
3523 }
3524 void setSuccessor(unsigned i, BasicBlock *NewSucc) {
3525 setOperand(i + 1, NewSucc);
3526 }
3527
3528 // Methods for support type inquiry through isa, cast, and dyn_cast:
3529 static bool classof(const Instruction *I) {
3530 return I->getOpcode() == Instruction::IndirectBr;
3531 }
3532 static bool classof(const Value *V) {
3533 return isa<Instruction>(V) && classof(cast<Instruction>(V));
3534 }
3535};
3536
3537template <>
3538struct OperandTraits<IndirectBrInst> : public HungoffOperandTraits<1> {
3539};
3540
3541DEFINE_TRANSPARENT_OPERAND_ACCESSORS(IndirectBrInst, Value)IndirectBrInst::op_iterator IndirectBrInst::op_begin() { return
OperandTraits<IndirectBrInst>::op_begin(this); } IndirectBrInst
::const_op_iterator IndirectBrInst::op_begin() const { return
OperandTraits<IndirectBrInst>::op_begin(const_cast<
IndirectBrInst*>(this)); } IndirectBrInst::op_iterator IndirectBrInst
::op_end() { return OperandTraits<IndirectBrInst>::op_end
(this); } IndirectBrInst::const_op_iterator IndirectBrInst::op_end
() const { return OperandTraits<IndirectBrInst>::op_end
(const_cast<IndirectBrInst*>(this)); } Value *IndirectBrInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<IndirectBrInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<IndirectBrInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3541, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<IndirectBrInst>::op_begin(
const_cast<IndirectBrInst*>(this))[i_nocapture].get());
} void IndirectBrInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<IndirectBrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<IndirectBrInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3541, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
IndirectBrInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned IndirectBrInst::getNumOperands() const { return OperandTraits
<IndirectBrInst>::operands(this); } template <int Idx_nocapture
> Use &IndirectBrInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &IndirectBrInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
3542
3543//===----------------------------------------------------------------------===//
3544// InvokeInst Class
3545//===----------------------------------------------------------------------===//
3546
3547/// Invoke instruction. The SubclassData field is used to hold the
3548/// calling convention of the call.
3549///
3550class InvokeInst : public TerminatorInst,
3551 public OperandBundleUser<InvokeInst, User::op_iterator> {
3552 friend class OperandBundleUser<InvokeInst, User::op_iterator>;
3553
3554 AttributeList Attrs;
3555 FunctionType *FTy;
3556
3557 InvokeInst(const InvokeInst &BI);
3558
3559 /// Construct an InvokeInst given a range of arguments.
3560 ///
3561 /// Construct an InvokeInst from a range of arguments
3562 inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
3563 ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
3564 unsigned Values, const Twine &NameStr,
3565 Instruction *InsertBefore)
3566 : InvokeInst(cast<FunctionType>(
3567 cast<PointerType>(Func->getType())->getElementType()),
3568 Func, IfNormal, IfException, Args, Bundles, Values, NameStr,
3569 InsertBefore) {}
3570
3571 inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3572 BasicBlock *IfException, ArrayRef<Value *> Args,
3573 ArrayRef<OperandBundleDef> Bundles, unsigned Values,
3574 const Twine &NameStr, Instruction *InsertBefore);
3575 /// Construct an InvokeInst given a range of arguments.
3576 ///
3577 /// Construct an InvokeInst from a range of arguments
3578 inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
3579 ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
3580 unsigned Values, const Twine &NameStr,
3581 BasicBlock *InsertAtEnd);
3582
3583 bool hasDescriptor() const { return HasDescriptor; }
3584
3585 void init(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
3586 ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
3587 const Twine &NameStr) {
3588 init(cast<FunctionType>(
3589 cast<PointerType>(Func->getType())->getElementType()),
3590 Func, IfNormal, IfException, Args, Bundles, NameStr);
3591 }
3592
3593 void init(FunctionType *FTy, Value *Func, BasicBlock *IfNormal,
3594 BasicBlock *IfException, ArrayRef<Value *> Args,
3595 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
3596
3597protected:
3598 // Note: Instruction needs to be a friend here to call cloneImpl.
3599 friend class Instruction;
3600
3601 InvokeInst *cloneImpl() const;
3602
3603public:
3604 static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
3605 BasicBlock *IfException, ArrayRef<Value *> Args,
3606 const Twine &NameStr,
3607 Instruction *InsertBefore = nullptr) {
3608 return Create(cast<FunctionType>(
3609 cast<PointerType>(Func->getType())->getElementType()),
3610 Func, IfNormal, IfException, Args, None, NameStr,
3611 InsertBefore);
3612 }
3613
3614 static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
3615 BasicBlock *IfException, ArrayRef<Value *> Args,
3616 ArrayRef<OperandBundleDef> Bundles = None,
3617 const Twine &NameStr = "",
3618 Instruction *InsertBefore = nullptr) {
3619 return Create(cast<FunctionType>(
3620 cast<PointerType>(Func->getType())->getElementType()),
3621 Func, IfNormal, IfException, Args, Bundles, NameStr,
3622 InsertBefore);
3623 }
3624
3625 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3626 BasicBlock *IfException, ArrayRef<Value *> Args,
3627 const Twine &NameStr,
3628 Instruction *InsertBefore = nullptr) {
3629 unsigned Values = unsigned(Args.size()) + 3;
3630 return new (Values) InvokeInst(Ty, Func, IfNormal, IfException, Args, None,
3631 Values, NameStr, InsertBefore);
3632 }
3633
3634 static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
3635 BasicBlock *IfException, ArrayRef<Value *> Args,
3636 ArrayRef<OperandBundleDef> Bundles = None,
3637 const Twine &NameStr = "",
3638 Instruction *InsertBefore = nullptr) {
3639 unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3;
3640 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3641
3642 return new (Values, DescriptorBytes)
3643 InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, Values,
3644 NameStr, InsertBefore);
3645 }
3646
3647 static InvokeInst *Create(Value *Func,
3648 BasicBlock *IfNormal, BasicBlock *IfException,
3649 ArrayRef<Value *> Args, const Twine &NameStr,
3650 BasicBlock *InsertAtEnd) {
3651 unsigned Values = unsigned(Args.size()) + 3;
3652 return new (Values) InvokeInst(Func, IfNormal, IfException, Args, None,
3653 Values, NameStr, InsertAtEnd);
3654 }
3655
3656 static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
3657 BasicBlock *IfException, ArrayRef<Value *> Args,
3658 ArrayRef<OperandBundleDef> Bundles,
3659 const Twine &NameStr, BasicBlock *InsertAtEnd) {
3660 unsigned Values = unsigned(Args.size()) + CountBundleInputs(Bundles) + 3;
3661 unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
3662
3663 return new (Values, DescriptorBytes)
3664 InvokeInst(Func, IfNormal, IfException, Args, Bundles, Values, NameStr,
3665 InsertAtEnd);
3666 }
3667
3668 /// Create a clone of \p II with a different set of operand bundles and
3669 /// insert it before \p InsertPt.
3670 ///
3671 /// The returned invoke instruction is identical to \p II in every way except
3672 /// that the operand bundles for the new instruction are set to the operand
3673 /// bundles in \p Bundles.
3674 static InvokeInst *Create(InvokeInst *II, ArrayRef<OperandBundleDef> Bundles,
3675 Instruction *InsertPt = nullptr);
3676
3677 /// Provide fast operand accessors
3678 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
3679
3680 FunctionType *getFunctionType() const { return FTy; }
3681
3682 void mutateFunctionType(FunctionType *FTy) {
3683 mutateType(FTy->getReturnType());
3684 this->FTy = FTy;
3685 }
3686
3687 /// Return the number of invoke arguments.
3688 ///
3689 unsigned getNumArgOperands() const {
3690 return getNumOperands() - getNumTotalBundleOperands() - 3;
3691 }
3692
3693 /// getArgOperand/setArgOperand - Return/set the i-th invoke argument.
3694 ///
3695 Value *getArgOperand(unsigned i) const {
3696 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3696, __extension__ __PRETTY_FUNCTION__));
3697 return getOperand(i);
3698 }
3699 void setArgOperand(unsigned i, Value *v) {
3700 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3700, __extension__ __PRETTY_FUNCTION__));
3701 setOperand(i, v);
3702 }
3703
3704 /// Return the iterator pointing to the beginning of the argument list.
3705 op_iterator arg_begin() { return op_begin(); }
3706
3707 /// Return the iterator pointing to the end of the argument list.
3708 op_iterator arg_end() {
3709 // [ invoke args ], [ operand bundles ], normal dest, unwind dest, callee
3710 return op_end() - getNumTotalBundleOperands() - 3;
3711 }
3712
3713 /// Iteration adapter for range-for loops.
3714 iterator_range<op_iterator> arg_operands() {
3715 return make_range(arg_begin(), arg_end());
3716 }
3717
3718 /// Return the iterator pointing to the beginning of the argument list.
3719 const_op_iterator arg_begin() const { return op_begin(); }
3720
3721 /// Return the iterator pointing to the end of the argument list.
3722 const_op_iterator arg_end() const {
3723 // [ invoke args ], [ operand bundles ], normal dest, unwind dest, callee
3724 return op_end() - getNumTotalBundleOperands() - 3;
3725 }
3726
3727 /// Iteration adapter for range-for loops.
3728 iterator_range<const_op_iterator> arg_operands() const {
3729 return make_range(arg_begin(), arg_end());
3730 }
3731
3732 /// Wrappers for getting the \c Use of a invoke argument.
3733 const Use &getArgOperandUse(unsigned i) const {
3734 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3734, __extension__ __PRETTY_FUNCTION__));
3735 return getOperandUse(i);
3736 }
3737 Use &getArgOperandUse(unsigned i) {
3738 assert(i < getNumArgOperands() && "Out of bounds!")(static_cast <bool> (i < getNumArgOperands() &&
"Out of bounds!") ? void (0) : __assert_fail ("i < getNumArgOperands() && \"Out of bounds!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3738, __extension__ __PRETTY_FUNCTION__));
3739 return getOperandUse(i);
3740 }
3741
3742 /// If one of the arguments has the 'returned' attribute, return its
3743 /// operand value. Otherwise, return nullptr.
3744 Value *getReturnedArgOperand() const;
3745
3746 /// getCallingConv/setCallingConv - Get or set the calling convention of this
3747 /// function call.
3748 CallingConv::ID getCallingConv() const {
3749 return static_cast<CallingConv::ID>(getSubclassDataFromInstruction());
3750 }
3751 void setCallingConv(CallingConv::ID CC) {
3752 auto ID = static_cast<unsigned>(CC);
3753 assert(!(ID & ~CallingConv::MaxID) && "Unsupported calling convention")(static_cast <bool> (!(ID & ~CallingConv::MaxID) &&
"Unsupported calling convention") ? void (0) : __assert_fail
("!(ID & ~CallingConv::MaxID) && \"Unsupported calling convention\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3753, __extension__ __PRETTY_FUNCTION__));
3754 setInstructionSubclassData(ID);
3755 }
3756
3757 /// Return the parameter attributes for this invoke.
3758 ///
3759 AttributeList getAttributes() const { return Attrs; }
3760
3761 /// Set the parameter attributes for this invoke.
3762 ///
3763 void setAttributes(AttributeList A) { Attrs = A; }
3764
3765 /// adds the attribute to the list of attributes.
3766 void addAttribute(unsigned i, Attribute::AttrKind Kind);
3767
3768 /// adds the attribute to the list of attributes.
3769 void addAttribute(unsigned i, Attribute Attr);
3770
3771 /// Adds the attribute to the indicated argument
3772 void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind);
3773
3774 /// removes the attribute from the list of attributes.
3775 void removeAttribute(unsigned i, Attribute::AttrKind Kind);
3776
3777 /// removes the attribute from the list of attributes.
3778 void removeAttribute(unsigned i, StringRef Kind);
3779
3780 /// Removes the attribute from the given argument
3781 void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind);
3782
3783 /// adds the dereferenceable attribute to the list of attributes.
3784 void addDereferenceableAttr(unsigned i, uint64_t Bytes);
3785
3786 /// adds the dereferenceable_or_null attribute to the list of
3787 /// attributes.
3788 void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes);
3789
3790 /// Determine whether this call has the given attribute.
3791 bool hasFnAttr(Attribute::AttrKind Kind) const {
3792 assert(Kind != Attribute::NoBuiltin &&(static_cast <bool> (Kind != Attribute::NoBuiltin &&
"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? void (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3793, __extension__ __PRETTY_FUNCTION__))
3793 "Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin")(static_cast <bool> (Kind != Attribute::NoBuiltin &&
"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin"
) ? void (0) : __assert_fail ("Kind != Attribute::NoBuiltin && \"Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3793, __extension__ __PRETTY_FUNCTION__));
3794 return hasFnAttrImpl(Kind);
3795 }
3796
3797 /// Determine whether this call has the given attribute.
3798 bool hasFnAttr(StringRef Kind) const {
3799 return hasFnAttrImpl(Kind);
3800 }
3801
3802 /// Determine whether the return value has the given attribute.
3803 bool hasRetAttr(Attribute::AttrKind Kind) const;
3804
3805 /// Determine whether the argument or parameter has the given attribute.
3806 bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
3807
3808 /// Get the attribute of a given kind at a position.
3809 Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const {
3810 return getAttributes().getAttribute(i, Kind);
3811 }
3812
3813 /// Get the attribute of a given kind at a position.
3814 Attribute getAttribute(unsigned i, StringRef Kind) const {
3815 return getAttributes().getAttribute(i, Kind);
3816 }
3817
3818 /// Return true if the data operand at index \p i has the attribute \p
3819 /// A.
3820 ///
3821 /// Data operands include invoke arguments and values used in operand bundles,
3822 /// but does not include the invokee operand, or the two successor blocks.
3823 /// This routine dispatches to the underlying AttributeList or the
3824 /// OperandBundleUser as appropriate.
3825 ///
3826 /// The index \p i is interpreted as
3827 ///
3828 /// \p i == Attribute::ReturnIndex -> the return value
3829 /// \p i in [1, arg_size + 1) -> argument number (\p i - 1)
3830 /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index
3831 /// (\p i - 1) in the operand list.
3832 bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const;
3833
3834 /// Extract the alignment of the return value.
3835 unsigned getRetAlignment() const { return Attrs.getRetAlignment(); }
3836
3837 /// Extract the alignment for a call or parameter (0=unknown).
3838 unsigned getParamAlignment(unsigned ArgNo) const {
3839 return Attrs.getParamAlignment(ArgNo);
3840 }
3841
3842 /// Extract the number of dereferenceable bytes for a call or
3843 /// parameter (0=unknown).
3844 uint64_t getDereferenceableBytes(unsigned i) const {
3845 return Attrs.getDereferenceableBytes(i);
3846 }
3847
3848 /// Extract the number of dereferenceable_or_null bytes for a call or
3849 /// parameter (0=unknown).
3850 uint64_t getDereferenceableOrNullBytes(unsigned i) const {
3851 return Attrs.getDereferenceableOrNullBytes(i);
3852 }
3853
3854 /// @brief Determine if the return value is marked with NoAlias attribute.
3855 bool returnDoesNotAlias() const {
3856 return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
3857 }
3858
3859 /// Return true if the call should not be treated as a call to a
3860 /// builtin.
3861 bool isNoBuiltin() const {
3862 // We assert in hasFnAttr if one passes in Attribute::NoBuiltin, so we have
3863 // to check it by hand.
3864 return hasFnAttrImpl(Attribute::NoBuiltin) &&
3865 !hasFnAttrImpl(Attribute::Builtin);
3866 }
3867
3868 /// Determine if the call requires strict floating point semantics.
3869 bool isStrictFP() const { return hasFnAttr(Attribute::StrictFP); }
3870
3871 /// Return true if the call should not be inlined.
3872 bool isNoInline() const { return hasFnAttr(Attribute::NoInline); }
3873 void setIsNoInline() {
3874 addAttribute(AttributeList::FunctionIndex, Attribute::NoInline);
3875 }
3876
3877 /// Determine if the call does not access memory.
3878 bool doesNotAccessMemory() const {
3879 return hasFnAttr(Attribute::ReadNone);
3880 }
3881 void setDoesNotAccessMemory() {
3882 addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
3883 }
3884
3885 /// Determine if the call does not access or only reads memory.
3886 bool onlyReadsMemory() const {
3887 return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly);
3888 }
3889 void setOnlyReadsMemory() {
3890 addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly);
3891 }
3892
3893 /// Determine if the call does not access or only writes memory.
3894 bool doesNotReadMemory() const {
3895 return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly);
3896 }
3897 void setDoesNotReadMemory() {
3898 addAttribute(AttributeList::FunctionIndex, Attribute::WriteOnly);
3899 }
3900
3901 /// @brief Determine if the call access memmory only using it's pointer
3902 /// arguments.
3903 bool onlyAccessesArgMemory() const {
3904 return hasFnAttr(Attribute::ArgMemOnly);
3905 }
3906 void setOnlyAccessesArgMemory() {
3907 addAttribute(AttributeList::FunctionIndex, Attribute::ArgMemOnly);
3908 }
3909
3910 /// @brief Determine if the function may only access memory that is
3911 /// inaccessible from the IR.
3912 bool onlyAccessesInaccessibleMemory() const {
3913 return hasFnAttr(Attribute::InaccessibleMemOnly);
3914 }
3915 void setOnlyAccessesInaccessibleMemory() {
3916 addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOnly);
3917 }
3918
3919 /// @brief Determine if the function may only access memory that is
3920 /// either inaccessible from the IR or pointed to by its arguments.
3921 bool onlyAccessesInaccessibleMemOrArgMem() const {
3922 return hasFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
3923 }
3924 void setOnlyAccessesInaccessibleMemOrArgMem() {
3925 addAttribute(AttributeList::FunctionIndex, Attribute::InaccessibleMemOrArgMemOnly);
3926 }
3927
3928 /// Determine if the call cannot return.
3929 bool doesNotReturn() const { return hasFnAttr(Attribute::NoReturn); }
3930 void setDoesNotReturn() {
3931 addAttribute(AttributeList::FunctionIndex, Attribute::NoReturn);
3932 }
3933
3934 /// Determine if the call cannot unwind.
3935 bool doesNotThrow() const { return hasFnAttr(Attribute::NoUnwind); }
3936 void setDoesNotThrow() {
3937 addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
3938 }
3939
3940 /// Determine if the invoke cannot be duplicated.
3941 bool cannotDuplicate() const {return hasFnAttr(Attribute::NoDuplicate); }
3942 void setCannotDuplicate() {
3943 addAttribute(AttributeList::FunctionIndex, Attribute::NoDuplicate);
3944 }
3945
3946 /// Determine if the invoke is convergent
3947 bool isConvergent() const { return hasFnAttr(Attribute::Convergent); }
3948 void setConvergent() {
3949 addAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
3950 }
3951 void setNotConvergent() {
3952 removeAttribute(AttributeList::FunctionIndex, Attribute::Convergent);
3953 }
3954
3955 /// Determine if the call returns a structure through first
3956 /// pointer argument.
3957 bool hasStructRetAttr() const {
3958 if (getNumArgOperands() == 0)
3959 return false;
3960
3961 // Be friendly and also check the callee.
3962 return paramHasAttr(0, Attribute::StructRet);
3963 }
3964
3965 /// Determine if any call argument is an aggregate passed by value.
3966 bool hasByValArgument() const {
3967 return Attrs.hasAttrSomewhere(Attribute::ByVal);
3968 }
3969
3970 /// Return the function called, or null if this is an
3971 /// indirect function invocation.
3972 ///
3973 Function *getCalledFunction() const {
3974 return dyn_cast<Function>(Op<-3>());
3975 }
3976
3977 /// Get a pointer to the function that is invoked by this
3978 /// instruction
3979 const Value *getCalledValue() const { return Op<-3>(); }
3980 Value *getCalledValue() { return Op<-3>(); }
3981
3982 /// Set the function called.
3983 void setCalledFunction(Value* Fn) {
3984 setCalledFunction(
3985 cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType()),
3986 Fn);
3987 }
3988 void setCalledFunction(FunctionType *FTy, Value *Fn) {
3989 this->FTy = FTy;
3990 assert(FTy == cast<FunctionType>((static_cast <bool> (FTy == cast<FunctionType>( cast
<PointerType>(Fn->getType())->getElementType())) ?
void (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3991, __extension__ __PRETTY_FUNCTION__))
3991 cast<PointerType>(Fn->getType())->getElementType()))(static_cast <bool> (FTy == cast<FunctionType>( cast
<PointerType>(Fn->getType())->getElementType())) ?
void (0) : __assert_fail ("FTy == cast<FunctionType>( cast<PointerType>(Fn->getType())->getElementType())"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 3991, __extension__ __PRETTY_FUNCTION__));
3992 Op<-3>() = Fn;
3993 }
3994
3995 // get*Dest - Return the destination basic blocks...
3996 BasicBlock *getNormalDest() const {
3997 return cast<BasicBlock>(Op<-2>());
3998 }
3999 BasicBlock *getUnwindDest() const {
4000 return cast<BasicBlock>(Op<-1>());
4001 }
4002 void setNormalDest(BasicBlock *B) {
4003 Op<-2>() = reinterpret_cast<Value*>(B);
4004 }
4005 void setUnwindDest(BasicBlock *B) {
4006 Op<-1>() = reinterpret_cast<Value*>(B);
4007 }
4008
4009 /// Get the landingpad instruction from the landing pad
4010 /// block (the unwind destination).
4011 LandingPadInst *getLandingPadInst() const;
4012
4013 BasicBlock *getSuccessor(unsigned i) const {
4014 assert(i < 2 && "Successor # out of range for invoke!")(static_cast <bool> (i < 2 && "Successor # out of range for invoke!"
) ? void (0) : __assert_fail ("i < 2 && \"Successor # out of range for invoke!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4014, __extension__ __PRETTY_FUNCTION__));
4015 return i == 0 ? getNormalDest() : getUnwindDest();
4016 }
4017
4018 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
4019 assert(idx < 2 && "Successor # out of range for invoke!")(static_cast <bool> (idx < 2 && "Successor # out of range for invoke!"
) ? void (0) : __assert_fail ("idx < 2 && \"Successor # out of range for invoke!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4019, __extension__ __PRETTY_FUNCTION__));
4020 *(&Op<-2>() + idx) = reinterpret_cast<Value*>(NewSucc);
4021 }
4022
4023 unsigned getNumSuccessors() const { return 2; }
4024
4025 // Methods for support type inquiry through isa, cast, and dyn_cast:
4026 static bool classof(const Instruction *I) {
4027 return (I->getOpcode() == Instruction::Invoke);
4028 }
4029 static bool classof(const Value *V) {
4030 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4031 }
4032
4033private:
4034 template <typename AttrKind> bool hasFnAttrImpl(AttrKind Kind) const {
4035 if (Attrs.hasAttribute(AttributeList::FunctionIndex, Kind))
4036 return true;
4037
4038 // Operand bundles override attributes on the called function, but don't
4039 // override attributes directly present on the invoke instruction.
4040 if (isFnAttrDisallowedByOpBundle(Kind))
4041 return false;
4042
4043 if (const Function *F = getCalledFunction())
4044 return F->getAttributes().hasAttribute(AttributeList::FunctionIndex,
4045 Kind);
4046 return false;
4047 }
4048
4049 // Shadow Instruction::setInstructionSubclassData with a private forwarding
4050 // method so that subclasses cannot accidentally use it.
4051 void setInstructionSubclassData(unsigned short D) {
4052 Instruction::setInstructionSubclassData(D);
4053 }
4054};
4055
4056template <>
4057struct OperandTraits<InvokeInst> : public VariadicOperandTraits<InvokeInst, 3> {
4058};
4059
4060InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
4061 BasicBlock *IfException, ArrayRef<Value *> Args,
4062 ArrayRef<OperandBundleDef> Bundles, unsigned Values,
4063 const Twine &NameStr, Instruction *InsertBefore)
4064 : TerminatorInst(Ty->getReturnType(), Instruction::Invoke,
4065 OperandTraits<InvokeInst>::op_end(this) - Values, Values,
4066 InsertBefore) {
4067 init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
4068}
4069
4070InvokeInst::InvokeInst(Value *Func, BasicBlock *IfNormal,
4071 BasicBlock *IfException, ArrayRef<Value *> Args,
4072 ArrayRef<OperandBundleDef> Bundles, unsigned Values,
4073 const Twine &NameStr, BasicBlock *InsertAtEnd)
4074 : TerminatorInst(
4075 cast<FunctionType>(cast<PointerType>(Func->getType())
4076 ->getElementType())->getReturnType(),
4077 Instruction::Invoke, OperandTraits<InvokeInst>::op_end(this) - Values,
4078 Values, InsertAtEnd) {
4079 init(Func, IfNormal, IfException, Args, Bundles, NameStr);
4080}
4081
4082DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InvokeInst, Value)InvokeInst::op_iterator InvokeInst::op_begin() { return OperandTraits
<InvokeInst>::op_begin(this); } InvokeInst::const_op_iterator
InvokeInst::op_begin() const { return OperandTraits<InvokeInst
>::op_begin(const_cast<InvokeInst*>(this)); } InvokeInst
::op_iterator InvokeInst::op_end() { return OperandTraits<
InvokeInst>::op_end(this); } InvokeInst::const_op_iterator
InvokeInst::op_end() const { return OperandTraits<InvokeInst
>::op_end(const_cast<InvokeInst*>(this)); } Value *InvokeInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<InvokeInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<InvokeInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4082, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InvokeInst>::op_begin(const_cast
<InvokeInst*>(this))[i_nocapture].get()); } void InvokeInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<InvokeInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<InvokeInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4082, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InvokeInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned InvokeInst::getNumOperands() const { return OperandTraits
<InvokeInst>::operands(this); } template <int Idx_nocapture
> Use &InvokeInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
InvokeInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
4083
4084//===----------------------------------------------------------------------===//
4085// ResumeInst Class
4086//===----------------------------------------------------------------------===//
4087
4088//===---------------------------------------------------------------------------
4089/// Resume the propagation of an exception.
4090///
4091class ResumeInst : public TerminatorInst {
4092 ResumeInst(const ResumeInst &RI);
4093
4094 explicit ResumeInst(Value *Exn, Instruction *InsertBefore=nullptr);
4095 ResumeInst(Value *Exn, BasicBlock *InsertAtEnd);
4096
4097protected:
4098 // Note: Instruction needs to be a friend here to call cloneImpl.
4099 friend class Instruction;
4100
4101 ResumeInst *cloneImpl() const;
4102
4103public:
4104 static ResumeInst *Create(Value *Exn, Instruction *InsertBefore = nullptr) {
4105 return new(1) ResumeInst(Exn, InsertBefore);
4106 }
4107
4108 static ResumeInst *Create(Value *Exn, BasicBlock *InsertAtEnd) {
4109 return new(1) ResumeInst(Exn, InsertAtEnd);
4110 }
4111
4112 /// Provide fast operand accessors
4113 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
4114
4115 /// Convenience accessor.
4116 Value *getValue() const { return Op<0>(); }
4117
4118 unsigned getNumSuccessors() const { return 0; }
4119
4120 // Methods for support type inquiry through isa, cast, and dyn_cast:
4121 static bool classof(const Instruction *I) {
4122 return I->getOpcode() == Instruction::Resume;
4123 }
4124 static bool classof(const Value *V) {
4125 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4126 }
4127
4128private:
4129 friend TerminatorInst;
4130
4131 BasicBlock *getSuccessor(unsigned idx) const {
4132 llvm_unreachable("ResumeInst has no successors!")::llvm::llvm_unreachable_internal("ResumeInst has no successors!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4132);
4133 }
4134
4135 void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
4136 llvm_unreachable("ResumeInst has no successors!")::llvm::llvm_unreachable_internal("ResumeInst has no successors!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4136);
4137 }
4138};
4139
4140template <>
4141struct OperandTraits<ResumeInst> :
4142 public FixedNumOperandTraits<ResumeInst, 1> {
4143};
4144
4145DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ResumeInst, Value)ResumeInst::op_iterator ResumeInst::op_begin() { return OperandTraits
<ResumeInst>::op_begin(this); } ResumeInst::const_op_iterator
ResumeInst::op_begin() const { return OperandTraits<ResumeInst
>::op_begin(const_cast<ResumeInst*>(this)); } ResumeInst
::op_iterator ResumeInst::op_end() { return OperandTraits<
ResumeInst>::op_end(this); } ResumeInst::const_op_iterator
ResumeInst::op_end() const { return OperandTraits<ResumeInst
>::op_end(const_cast<ResumeInst*>(this)); } Value *ResumeInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<ResumeInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ResumeInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4145, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ResumeInst>::op_begin(const_cast
<ResumeInst*>(this))[i_nocapture].get()); } void ResumeInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<ResumeInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<ResumeInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4145, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ResumeInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned ResumeInst::getNumOperands() const { return OperandTraits
<ResumeInst>::operands(this); } template <int Idx_nocapture
> Use &ResumeInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
ResumeInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
4146
4147//===----------------------------------------------------------------------===//
4148// CatchSwitchInst Class
4149//===----------------------------------------------------------------------===//
4150class CatchSwitchInst : public TerminatorInst {
4151 /// The number of operands actually allocated. NumOperands is
4152 /// the number actually in use.
4153 unsigned ReservedSpace;
4154
4155 // Operand[0] = Outer scope
4156 // Operand[1] = Unwind block destination
4157 // Operand[n] = BasicBlock to go to on match
4158 CatchSwitchInst(const CatchSwitchInst &CSI);
4159
4160 /// Create a new switch instruction, specifying a
4161 /// default destination. The number of additional handlers can be specified
4162 /// here to make memory allocation more efficient.
4163 /// This constructor can also autoinsert before another instruction.
4164 CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
4165 unsigned NumHandlers, const Twine &NameStr,
4166 Instruction *InsertBefore);
4167
4168 /// Create a new switch instruction, specifying a
4169 /// default destination. The number of additional handlers can be specified
4170 /// here to make memory allocation more efficient.
4171 /// This constructor also autoinserts at the end of the specified BasicBlock.
4172 CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
4173 unsigned NumHandlers, const Twine &NameStr,
4174 BasicBlock *InsertAtEnd);
4175
4176 // allocate space for exactly zero operands
4177 void *operator new(size_t s) { return User::operator new(s); }
4178
4179 void init(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumReserved);
4180 void growOperands(unsigned Size);
4181
4182protected:
4183 // Note: Instruction needs to be a friend here to call cloneImpl.
4184 friend class Instruction;
4185
4186 CatchSwitchInst *cloneImpl() const;
4187
4188public:
4189 static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest,
4190 unsigned NumHandlers,
4191 const Twine &NameStr = "",
4192 Instruction *InsertBefore = nullptr) {
4193 return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr,
4194 InsertBefore);
4195 }
4196
4197 static CatchSwitchInst *Create(Value *ParentPad, BasicBlock *UnwindDest,
4198 unsigned NumHandlers, const Twine &NameStr,
4199 BasicBlock *InsertAtEnd) {
4200 return new CatchSwitchInst(ParentPad, UnwindDest, NumHandlers, NameStr,
4201 InsertAtEnd);
4202 }
4203
4204 /// Provide fast operand accessors
4205 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
4206
4207 // Accessor Methods for CatchSwitch stmt
4208 Value *getParentPad() const { return getOperand(0); }
4209 void setParentPad(Value *ParentPad) { setOperand(0, ParentPad); }
4210
4211 // Accessor Methods for CatchSwitch stmt
4212 bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
4213 bool unwindsToCaller() const { return !hasUnwindDest(); }
4214 BasicBlock *getUnwindDest() const {
4215 if (hasUnwindDest())
4216 return cast<BasicBlock>(getOperand(1));
4217 return nullptr;
4218 }
4219 void setUnwindDest(BasicBlock *UnwindDest) {
4220 assert(UnwindDest)(static_cast <bool> (UnwindDest) ? void (0) : __assert_fail
("UnwindDest", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4220, __extension__ __PRETTY_FUNCTION__));
4221 assert(hasUnwindDest())(static_cast <bool> (hasUnwindDest()) ? void (0) : __assert_fail
("hasUnwindDest()", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4221, __extension__ __PRETTY_FUNCTION__));
4222 setOperand(1, UnwindDest);
4223 }
4224
4225 /// return the number of 'handlers' in this catchswitch
4226 /// instruction, except the default handler
4227 unsigned getNumHandlers() const {
4228 if (hasUnwindDest())
4229 return getNumOperands() - 2;
4230 return getNumOperands() - 1;
4231 }
4232
4233private:
4234 static BasicBlock *handler_helper(Value *V) { return cast<BasicBlock>(V); }
4235 static const BasicBlock *handler_helper(const Value *V) {
4236 return cast<BasicBlock>(V);
4237 }
4238
4239public:
4240 using DerefFnTy = BasicBlock *(*)(Value *);
4241 using handler_iterator = mapped_iterator<op_iterator, DerefFnTy>;
4242 using handler_range = iterator_range<handler_iterator>;
4243 using ConstDerefFnTy = const BasicBlock *(*)(const Value *);
4244 using const_handler_iterator =
4245 mapped_iterator<const_op_iterator, ConstDerefFnTy>;
4246 using const_handler_range = iterator_range<const_handler_iterator>;
4247
4248 /// Returns an iterator that points to the first handler in CatchSwitchInst.
4249 handler_iterator handler_begin() {
4250 op_iterator It = op_begin() + 1;
4251 if (hasUnwindDest())
4252 ++It;
4253 return handler_iterator(It, DerefFnTy(handler_helper));
4254 }
4255
4256 /// Returns an iterator that points to the first handler in the
4257 /// CatchSwitchInst.
4258 const_handler_iterator handler_begin() const {
4259 const_op_iterator It = op_begin() + 1;
4260 if (hasUnwindDest())
4261 ++It;
4262 return const_handler_iterator(It, ConstDerefFnTy(handler_helper));
4263 }
4264
4265 /// Returns a read-only iterator that points one past the last
4266 /// handler in the CatchSwitchInst.
4267 handler_iterator handler_end() {
4268 return handler_iterator(op_end(), DerefFnTy(handler_helper));
4269 }
4270
4271 /// Returns an iterator that points one past the last handler in the
4272 /// CatchSwitchInst.
4273 const_handler_iterator handler_end() const {
4274 return const_handler_iterator(op_end(), ConstDerefFnTy(handler_helper));
4275 }
4276
4277 /// iteration adapter for range-for loops.
4278 handler_range handlers() {
4279 return make_range(handler_begin(), handler_end());
4280 }
4281
4282 /// iteration adapter for range-for loops.
4283 const_handler_range handlers() const {
4284 return make_range(handler_begin(), handler_end());
4285 }
4286
4287 /// Add an entry to the switch instruction...
4288 /// Note:
4289 /// This action invalidates handler_end(). Old handler_end() iterator will
4290 /// point to the added handler.
4291 void addHandler(BasicBlock *Dest);
4292
4293 void removeHandler(handler_iterator HI);
4294
4295 unsigned getNumSuccessors() const { return getNumOperands() - 1; }
4296 BasicBlock *getSuccessor(unsigned Idx) const {
4297 assert(Idx < getNumSuccessors() &&(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4298, __extension__ __PRETTY_FUNCTION__))
4298 "Successor # out of range for catchswitch!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4298, __extension__ __PRETTY_FUNCTION__));
4299 return cast<BasicBlock>(getOperand(Idx + 1));
4300 }
4301 void setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
4302 assert(Idx < getNumSuccessors() &&(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4303, __extension__ __PRETTY_FUNCTION__))
4303 "Successor # out of range for catchswitch!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchswitch!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchswitch!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4303, __extension__ __PRETTY_FUNCTION__));
4304 setOperand(Idx + 1, NewSucc);
4305 }
4306
4307 // Methods for support type inquiry through isa, cast, and dyn_cast:
4308 static bool classof(const Instruction *I) {
4309 return I->getOpcode() == Instruction::CatchSwitch;
4310 }
4311 static bool classof(const Value *V) {
4312 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4313 }
4314};
4315
4316template <>
4317struct OperandTraits<CatchSwitchInst> : public HungoffOperandTraits<2> {};
4318
4319DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchSwitchInst, Value)CatchSwitchInst::op_iterator CatchSwitchInst::op_begin() { return
OperandTraits<CatchSwitchInst>::op_begin(this); } CatchSwitchInst
::const_op_iterator CatchSwitchInst::op_begin() const { return
OperandTraits<CatchSwitchInst>::op_begin(const_cast<
CatchSwitchInst*>(this)); } CatchSwitchInst::op_iterator CatchSwitchInst
::op_end() { return OperandTraits<CatchSwitchInst>::op_end
(this); } CatchSwitchInst::const_op_iterator CatchSwitchInst::
op_end() const { return OperandTraits<CatchSwitchInst>::
op_end(const_cast<CatchSwitchInst*>(this)); } Value *CatchSwitchInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<CatchSwitchInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<CatchSwitchInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4319, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<CatchSwitchInst>::op_begin
(const_cast<CatchSwitchInst*>(this))[i_nocapture].get()
); } void CatchSwitchInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<CatchSwitchInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CatchSwitchInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4319, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
CatchSwitchInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned CatchSwitchInst::getNumOperands() const { return
OperandTraits<CatchSwitchInst>::operands(this); } template
<int Idx_nocapture> Use &CatchSwitchInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &CatchSwitchInst::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
4320
4321//===----------------------------------------------------------------------===//
4322// CleanupPadInst Class
4323//===----------------------------------------------------------------------===//
4324class CleanupPadInst : public FuncletPadInst {
4325private:
4326 explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args,
4327 unsigned Values, const Twine &NameStr,
4328 Instruction *InsertBefore)
4329 : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values,
4330 NameStr, InsertBefore) {}
4331 explicit CleanupPadInst(Value *ParentPad, ArrayRef<Value *> Args,
4332 unsigned Values, const Twine &NameStr,
4333 BasicBlock *InsertAtEnd)
4334 : FuncletPadInst(Instruction::CleanupPad, ParentPad, Args, Values,
4335 NameStr, InsertAtEnd) {}
4336
4337public:
4338 static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args = None,
4339 const Twine &NameStr = "",
4340 Instruction *InsertBefore = nullptr) {
4341 unsigned Values = 1 + Args.size();
4342 return new (Values)
4343 CleanupPadInst(ParentPad, Args, Values, NameStr, InsertBefore);
4344 }
4345
4346 static CleanupPadInst *Create(Value *ParentPad, ArrayRef<Value *> Args,
4347 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4348 unsigned Values = 1 + Args.size();
4349 return new (Values)
4350 CleanupPadInst(ParentPad, Args, Values, NameStr, InsertAtEnd);
4351 }
4352
4353 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4354 static bool classof(const Instruction *I) {
4355 return I->getOpcode() == Instruction::CleanupPad;
4356 }
4357 static bool classof(const Value *V) {
4358 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4359 }
4360};
4361
4362//===----------------------------------------------------------------------===//
4363// CatchPadInst Class
4364//===----------------------------------------------------------------------===//
4365class CatchPadInst : public FuncletPadInst {
4366private:
4367 explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args,
4368 unsigned Values, const Twine &NameStr,
4369 Instruction *InsertBefore)
4370 : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values,
4371 NameStr, InsertBefore) {}
4372 explicit CatchPadInst(Value *CatchSwitch, ArrayRef<Value *> Args,
4373 unsigned Values, const Twine &NameStr,
4374 BasicBlock *InsertAtEnd)
4375 : FuncletPadInst(Instruction::CatchPad, CatchSwitch, Args, Values,
4376 NameStr, InsertAtEnd) {}
4377
4378public:
4379 static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args,
4380 const Twine &NameStr = "",
4381 Instruction *InsertBefore = nullptr) {
4382 unsigned Values = 1 + Args.size();
4383 return new (Values)
4384 CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertBefore);
4385 }
4386
4387 static CatchPadInst *Create(Value *CatchSwitch, ArrayRef<Value *> Args,
4388 const Twine &NameStr, BasicBlock *InsertAtEnd) {
4389 unsigned Values = 1 + Args.size();
4390 return new (Values)
4391 CatchPadInst(CatchSwitch, Args, Values, NameStr, InsertAtEnd);
4392 }
4393
4394 /// Convenience accessors
4395 CatchSwitchInst *getCatchSwitch() const {
4396 return cast<CatchSwitchInst>(Op<-1>());
4397 }
4398 void setCatchSwitch(Value *CatchSwitch) {
4399 assert(CatchSwitch)(static_cast <bool> (CatchSwitch) ? void (0) : __assert_fail
("CatchSwitch", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4399, __extension__ __PRETTY_FUNCTION__));
4400 Op<-1>() = CatchSwitch;
4401 }
4402
4403 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4404 static bool classof(const Instruction *I) {
4405 return I->getOpcode() == Instruction::CatchPad;
4406 }
4407 static bool classof(const Value *V) {
4408 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4409 }
4410};
4411
4412//===----------------------------------------------------------------------===//
4413// CatchReturnInst Class
4414//===----------------------------------------------------------------------===//
4415
4416class CatchReturnInst : public TerminatorInst {
4417 CatchReturnInst(const CatchReturnInst &RI);
4418 CatchReturnInst(Value *CatchPad, BasicBlock *BB, Instruction *InsertBefore);
4419 CatchReturnInst(Value *CatchPad, BasicBlock *BB, BasicBlock *InsertAtEnd);
4420
4421 void init(Value *CatchPad, BasicBlock *BB);
4422
4423protected:
4424 // Note: Instruction needs to be a friend here to call cloneImpl.
4425 friend class Instruction;
4426
4427 CatchReturnInst *cloneImpl() const;
4428
4429public:
4430 static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
4431 Instruction *InsertBefore = nullptr) {
4432 assert(CatchPad)(static_cast <bool> (CatchPad) ? void (0) : __assert_fail
("CatchPad", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4432, __extension__ __PRETTY_FUNCTION__));
4433 assert(BB)(static_cast <bool> (BB) ? void (0) : __assert_fail ("BB"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4433, __extension__ __PRETTY_FUNCTION__));
4434 return new (2) CatchReturnInst(CatchPad, BB, InsertBefore);
4435 }
4436
4437 static CatchReturnInst *Create(Value *CatchPad, BasicBlock *BB,
4438 BasicBlock *InsertAtEnd) {
4439 assert(CatchPad)(static_cast <bool> (CatchPad) ? void (0) : __assert_fail
("CatchPad", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4439, __extension__ __PRETTY_FUNCTION__));
4440 assert(BB)(static_cast <bool> (BB) ? void (0) : __assert_fail ("BB"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4440, __extension__ __PRETTY_FUNCTION__));
4441 return new (2) CatchReturnInst(CatchPad, BB, InsertAtEnd);
4442 }
4443
4444 /// Provide fast operand accessors
4445 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
4446
4447 /// Convenience accessors.
4448 CatchPadInst *getCatchPad() const { return cast<CatchPadInst>(Op<0>()); }
4449 void setCatchPad(CatchPadInst *CatchPad) {
4450 assert(CatchPad)(static_cast <bool> (CatchPad) ? void (0) : __assert_fail
("CatchPad", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4450, __extension__ __PRETTY_FUNCTION__));
4451 Op<0>() = CatchPad;
4452 }
4453
4454 BasicBlock *getSuccessor() const { return cast<BasicBlock>(Op<1>()); }
4455 void setSuccessor(BasicBlock *NewSucc) {
4456 assert(NewSucc)(static_cast <bool> (NewSucc) ? void (0) : __assert_fail
("NewSucc", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4456, __extension__ __PRETTY_FUNCTION__));
4457 Op<1>() = NewSucc;
4458 }
4459 unsigned getNumSuccessors() const { return 1; }
4460
4461 /// Get the parentPad of this catchret's catchpad's catchswitch.
4462 /// The successor block is implicitly a member of this funclet.
4463 Value *getCatchSwitchParentPad() const {
4464 return getCatchPad()->getCatchSwitch()->getParentPad();
4465 }
4466
4467 // Methods for support type inquiry through isa, cast, and dyn_cast:
4468 static bool classof(const Instruction *I) {
4469 return (I->getOpcode() == Instruction::CatchRet);
4470 }
4471 static bool classof(const Value *V) {
4472 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4473 }
4474
4475private:
4476 friend TerminatorInst;
4477
4478 BasicBlock *getSuccessor(unsigned Idx) const {
4479 assert(Idx < getNumSuccessors() && "Successor # out of range for catchret!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchret!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchret!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4479, __extension__ __PRETTY_FUNCTION__));
4480 return getSuccessor();
4481 }
4482
4483 void setSuccessor(unsigned Idx, BasicBlock *B) {
4484 assert(Idx < getNumSuccessors() && "Successor # out of range for catchret!")(static_cast <bool> (Idx < getNumSuccessors() &&
"Successor # out of range for catchret!") ? void (0) : __assert_fail
("Idx < getNumSuccessors() && \"Successor # out of range for catchret!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4484, __extension__ __PRETTY_FUNCTION__));
4485 setSuccessor(B);
4486 }
4487};
4488
4489template <>
4490struct OperandTraits<CatchReturnInst>
4491 : public FixedNumOperandTraits<CatchReturnInst, 2> {};
4492
4493DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CatchReturnInst, Value)CatchReturnInst::op_iterator CatchReturnInst::op_begin() { return
OperandTraits<CatchReturnInst>::op_begin(this); } CatchReturnInst
::const_op_iterator CatchReturnInst::op_begin() const { return
OperandTraits<CatchReturnInst>::op_begin(const_cast<
CatchReturnInst*>(this)); } CatchReturnInst::op_iterator CatchReturnInst
::op_end() { return OperandTraits<CatchReturnInst>::op_end
(this); } CatchReturnInst::const_op_iterator CatchReturnInst::
op_end() const { return OperandTraits<CatchReturnInst>::
op_end(const_cast<CatchReturnInst*>(this)); } Value *CatchReturnInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<CatchReturnInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<CatchReturnInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4493, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<CatchReturnInst>::op_begin
(const_cast<CatchReturnInst*>(this))[i_nocapture].get()
); } void CatchReturnInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<CatchReturnInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CatchReturnInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4493, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
CatchReturnInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned CatchReturnInst::getNumOperands() const { return
OperandTraits<CatchReturnInst>::operands(this); } template
<int Idx_nocapture> Use &CatchReturnInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &CatchReturnInst::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
4494
4495//===----------------------------------------------------------------------===//
4496// CleanupReturnInst Class
4497//===----------------------------------------------------------------------===//
4498
4499class CleanupReturnInst : public TerminatorInst {
4500private:
4501 CleanupReturnInst(const CleanupReturnInst &RI);
4502 CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
4503 Instruction *InsertBefore = nullptr);
4504 CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB, unsigned Values,
4505 BasicBlock *InsertAtEnd);
4506
4507 void init(Value *CleanupPad, BasicBlock *UnwindBB);
4508
4509protected:
4510 // Note: Instruction needs to be a friend here to call cloneImpl.
4511 friend class Instruction;
4512
4513 CleanupReturnInst *cloneImpl() const;
4514
4515public:
4516 static CleanupReturnInst *Create(Value *CleanupPad,
4517 BasicBlock *UnwindBB = nullptr,
4518 Instruction *InsertBefore = nullptr) {
4519 assert(CleanupPad)(static_cast <bool> (CleanupPad) ? void (0) : __assert_fail
("CleanupPad", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4519, __extension__ __PRETTY_FUNCTION__));
4520 unsigned Values = 1;
4521 if (UnwindBB)
4522 ++Values;
4523 return new (Values)
4524 CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertBefore);
4525 }
4526
4527 static CleanupReturnInst *Create(Value *CleanupPad, BasicBlock *UnwindBB,
4528 BasicBlock *InsertAtEnd) {
4529 assert(CleanupPad)(static_cast <bool> (CleanupPad) ? void (0) : __assert_fail
("CleanupPad", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4529, __extension__ __PRETTY_FUNCTION__));
4530 unsigned Values = 1;
4531 if (UnwindBB)
4532 ++Values;
4533 return new (Values)
4534 CleanupReturnInst(CleanupPad, UnwindBB, Values, InsertAtEnd);
4535 }
4536
4537 /// Provide fast operand accessors
4538 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const;
4539
4540 bool hasUnwindDest() const { return getSubclassDataFromInstruction() & 1; }
4541 bool unwindsToCaller() const { return !hasUnwindDest(); }
4542
4543 /// Convenience accessor.
4544 CleanupPadInst *getCleanupPad() const {
4545 return cast<CleanupPadInst>(Op<0>());
4546 }
4547 void setCleanupPad(CleanupPadInst *CleanupPad) {
4548 assert(CleanupPad)(static_cast <bool> (CleanupPad) ? void (0) : __assert_fail
("CleanupPad", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4548, __extension__ __PRETTY_FUNCTION__));
4549 Op<0>() = CleanupPad;
4550 }
4551
4552 unsigned getNumSuccessors() const { return hasUnwindDest() ? 1 : 0; }
4553
4554 BasicBlock *getUnwindDest() const {
4555 return hasUnwindDest() ? cast<BasicBlock>(Op<1>()) : nullptr;
4556 }
4557 void setUnwindDest(BasicBlock *NewDest) {
4558 assert(NewDest)(static_cast <bool> (NewDest) ? void (0) : __assert_fail
("NewDest", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4558, __extension__ __PRETTY_FUNCTION__));
4559 assert(hasUnwindDest())(static_cast <bool> (hasUnwindDest()) ? void (0) : __assert_fail
("hasUnwindDest()", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4559, __extension__ __PRETTY_FUNCTION__));
4560 Op<1>() = NewDest;
4561 }
4562
4563 // Methods for support type inquiry through isa, cast, and dyn_cast:
4564 static bool classof(const Instruction *I) {
4565 return (I->getOpcode() == Instruction::CleanupRet);
4566 }
4567 static bool classof(const Value *V) {
4568 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4569 }
4570
4571private:
4572 friend TerminatorInst;
4573
4574 BasicBlock *getSuccessor(unsigned Idx) const {
4575 assert(Idx == 0)(static_cast <bool> (Idx == 0) ? void (0) : __assert_fail
("Idx == 0", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4575, __extension__ __PRETTY_FUNCTION__));
4576 return getUnwindDest();
4577 }
4578
4579 void setSuccessor(unsigned Idx, BasicBlock *B) {
4580 assert(Idx == 0)(static_cast <bool> (Idx == 0) ? void (0) : __assert_fail
("Idx == 0", "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4580, __extension__ __PRETTY_FUNCTION__));
4581 setUnwindDest(B);
4582 }
4583
4584 // Shadow Instruction::setInstructionSubclassData with a private forwarding
4585 // method so that subclasses cannot accidentally use it.
4586 void setInstructionSubclassData(unsigned short D) {
4587 Instruction::setInstructionSubclassData(D);
4588 }
4589};
4590
4591template <>
4592struct OperandTraits<CleanupReturnInst>
4593 : public VariadicOperandTraits<CleanupReturnInst, /*MINARITY=*/1> {};
4594
4595DEFINE_TRANSPARENT_OPERAND_ACCESSORS(CleanupReturnInst, Value)CleanupReturnInst::op_iterator CleanupReturnInst::op_begin() {
return OperandTraits<CleanupReturnInst>::op_begin(this
); } CleanupReturnInst::const_op_iterator CleanupReturnInst::
op_begin() const { return OperandTraits<CleanupReturnInst>
::op_begin(const_cast<CleanupReturnInst*>(this)); } CleanupReturnInst
::op_iterator CleanupReturnInst::op_end() { return OperandTraits
<CleanupReturnInst>::op_end(this); } CleanupReturnInst::
const_op_iterator CleanupReturnInst::op_end() const { return OperandTraits
<CleanupReturnInst>::op_end(const_cast<CleanupReturnInst
*>(this)); } Value *CleanupReturnInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<CleanupReturnInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CleanupReturnInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4595, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<CleanupReturnInst>::op_begin
(const_cast<CleanupReturnInst*>(this))[i_nocapture].get
()); } void CleanupReturnInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<CleanupReturnInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<CleanupReturnInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4595, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
CleanupReturnInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned CleanupReturnInst::getNumOperands() const { return
OperandTraits<CleanupReturnInst>::operands(this); } template
<int Idx_nocapture> Use &CleanupReturnInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &CleanupReturnInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
4596
4597//===----------------------------------------------------------------------===//
4598// UnreachableInst Class
4599//===----------------------------------------------------------------------===//
4600
4601//===---------------------------------------------------------------------------
4602/// This function has undefined behavior. In particular, the
4603/// presence of this instruction indicates some higher level knowledge that the
4604/// end of the block cannot be reached.
4605///
4606class UnreachableInst : public TerminatorInst {
4607protected:
4608 // Note: Instruction needs to be a friend here to call cloneImpl.
4609 friend class Instruction;
4610
4611 UnreachableInst *cloneImpl() const;
4612
4613public:
4614 explicit UnreachableInst(LLVMContext &C, Instruction *InsertBefore = nullptr);
4615 explicit UnreachableInst(LLVMContext &C, BasicBlock *InsertAtEnd);
4616
4617 // allocate space for exactly zero operands
4618 void *operator new(size_t s) {
4619 return User::operator new(s, 0);
4620 }
4621
4622 unsigned getNumSuccessors() const { return 0; }
4623
4624 // Methods for support type inquiry through isa, cast, and dyn_cast:
4625 static bool classof(const Instruction *I) {
4626 return I->getOpcode() == Instruction::Unreachable;
4627 }
4628 static bool classof(const Value *V) {
4629 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4630 }
4631
4632private:
4633 friend TerminatorInst;
4634
4635 BasicBlock *getSuccessor(unsigned idx) const {
4636 llvm_unreachable("UnreachableInst has no successors!")::llvm::llvm_unreachable_internal("UnreachableInst has no successors!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4636);
4637 }
4638
4639 void setSuccessor(unsigned idx, BasicBlock *B) {
4640 llvm_unreachable("UnreachableInst has no successors!")::llvm::llvm_unreachable_internal("UnreachableInst has no successors!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instructions.h"
, 4640);
4641 }
4642};
4643
4644//===----------------------------------------------------------------------===//
4645// TruncInst Class
4646//===----------------------------------------------------------------------===//
4647
4648/// This class represents a truncation of integer types.
4649class TruncInst : public CastInst {
4650protected:
4651 // Note: Instruction needs to be a friend here to call cloneImpl.
4652 friend class Instruction;
4653
4654 /// Clone an identical TruncInst
4655 TruncInst *cloneImpl() const;
4656
4657public:
4658 /// Constructor with insert-before-instruction semantics
4659 TruncInst(
4660 Value *S, ///< The value to be truncated
4661 Type *Ty, ///< The (smaller) type to truncate to
4662 const Twine &NameStr = "", ///< A name for the new instruction
4663 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4664 );
4665
4666 /// Constructor with insert-at-end-of-block semantics
4667 TruncInst(
4668 Value *S, ///< The value to be truncated
4669 Type *Ty, ///< The (smaller) type to truncate to
4670 const Twine &NameStr, ///< A name for the new instruction
4671 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4672 );
4673
4674 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4675 static bool classof(const Instruction *I) {
4676 return I->getOpcode() == Trunc;
4677 }
4678 static bool classof(const Value *V) {
4679 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4680 }
4681};
4682
4683//===----------------------------------------------------------------------===//
4684// ZExtInst Class
4685//===----------------------------------------------------------------------===//
4686
4687/// This class represents zero extension of integer types.
4688class ZExtInst : public CastInst {
4689protected:
4690 // Note: Instruction needs to be a friend here to call cloneImpl.
4691 friend class Instruction;
4692
4693 /// Clone an identical ZExtInst
4694 ZExtInst *cloneImpl() const;
4695
4696public:
4697 /// Constructor with insert-before-instruction semantics
4698 ZExtInst(
4699 Value *S, ///< The value to be zero extended
4700 Type *Ty, ///< The type to zero extend to
4701 const Twine &NameStr = "", ///< A name for the new instruction
4702 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4703 );
4704
4705 /// Constructor with insert-at-end semantics.
4706 ZExtInst(
4707 Value *S, ///< The value to be zero extended
4708 Type *Ty, ///< The type to zero extend to
4709 const Twine &NameStr, ///< A name for the new instruction
4710 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4711 );
4712
4713 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4714 static bool classof(const Instruction *I) {
4715 return I->getOpcode() == ZExt;
4716 }
4717 static bool classof(const Value *V) {
4718 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4719 }
4720};
4721
4722//===----------------------------------------------------------------------===//
4723// SExtInst Class
4724//===----------------------------------------------------------------------===//
4725
4726/// This class represents a sign extension of integer types.
4727class SExtInst : public CastInst {
4728protected:
4729 // Note: Instruction needs to be a friend here to call cloneImpl.
4730 friend class Instruction;
4731
4732 /// Clone an identical SExtInst
4733 SExtInst *cloneImpl() const;
4734
4735public:
4736 /// Constructor with insert-before-instruction semantics
4737 SExtInst(
4738 Value *S, ///< The value to be sign extended
4739 Type *Ty, ///< The type to sign extend to
4740 const Twine &NameStr = "", ///< A name for the new instruction
4741 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4742 );
4743
4744 /// Constructor with insert-at-end-of-block semantics
4745 SExtInst(
4746 Value *S, ///< The value to be sign extended
4747 Type *Ty, ///< The type to sign extend to
4748 const Twine &NameStr, ///< A name for the new instruction
4749 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4750 );
4751
4752 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4753 static bool classof(const Instruction *I) {
4754 return I->getOpcode() == SExt;
4755 }
4756 static bool classof(const Value *V) {
4757 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4758 }
4759};
4760
4761//===----------------------------------------------------------------------===//
4762// FPTruncInst Class
4763//===----------------------------------------------------------------------===//
4764
4765/// This class represents a truncation of floating point types.
4766class FPTruncInst : public CastInst {
4767protected:
4768 // Note: Instruction needs to be a friend here to call cloneImpl.
4769 friend class Instruction;
4770
4771 /// Clone an identical FPTruncInst
4772 FPTruncInst *cloneImpl() const;
4773
4774public:
4775 /// Constructor with insert-before-instruction semantics
4776 FPTruncInst(
4777 Value *S, ///< The value to be truncated
4778 Type *Ty, ///< The type to truncate to
4779 const Twine &NameStr = "", ///< A name for the new instruction
4780 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4781 );
4782
4783 /// Constructor with insert-before-instruction semantics
4784 FPTruncInst(
4785 Value *S, ///< The value to be truncated
4786 Type *Ty, ///< The type to truncate to
4787 const Twine &NameStr, ///< A name for the new instruction
4788 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4789 );
4790
4791 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4792 static bool classof(const Instruction *I) {
4793 return I->getOpcode() == FPTrunc;
4794 }
4795 static bool classof(const Value *V) {
4796 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4797 }
4798};
4799
4800//===----------------------------------------------------------------------===//
4801// FPExtInst Class
4802//===----------------------------------------------------------------------===//
4803
4804/// This class represents an extension of floating point types.
4805class FPExtInst : public CastInst {
4806protected:
4807 // Note: Instruction needs to be a friend here to call cloneImpl.
4808 friend class Instruction;
4809
4810 /// Clone an identical FPExtInst
4811 FPExtInst *cloneImpl() const;
4812
4813public:
4814 /// Constructor with insert-before-instruction semantics
4815 FPExtInst(
4816 Value *S, ///< The value to be extended
4817 Type *Ty, ///< The type to extend to
4818 const Twine &NameStr = "", ///< A name for the new instruction
4819 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4820 );
4821
4822 /// Constructor with insert-at-end-of-block semantics
4823 FPExtInst(
4824 Value *S, ///< The value to be extended
4825 Type *Ty, ///< The type to extend to
4826 const Twine &NameStr, ///< A name for the new instruction
4827 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4828 );
4829
4830 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4831 static bool classof(const Instruction *I) {
4832 return I->getOpcode() == FPExt;
4833 }
4834 static bool classof(const Value *V) {
4835 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4836 }
4837};
4838
4839//===----------------------------------------------------------------------===//
4840// UIToFPInst Class
4841//===----------------------------------------------------------------------===//
4842
4843/// This class represents a cast unsigned integer to floating point.
4844class UIToFPInst : public CastInst {
4845protected:
4846 // Note: Instruction needs to be a friend here to call cloneImpl.
4847 friend class Instruction;
4848
4849 /// Clone an identical UIToFPInst
4850 UIToFPInst *cloneImpl() const;
4851
4852public:
4853 /// Constructor with insert-before-instruction semantics
4854 UIToFPInst(
4855 Value *S, ///< The value to be converted
4856 Type *Ty, ///< The type to convert to
4857 const Twine &NameStr = "", ///< A name for the new instruction
4858 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4859 );
4860
4861 /// Constructor with insert-at-end-of-block semantics
4862 UIToFPInst(
4863 Value *S, ///< The value to be converted
4864 Type *Ty, ///< The type to convert to
4865 const Twine &NameStr, ///< A name for the new instruction
4866 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4867 );
4868
4869 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4870 static bool classof(const Instruction *I) {
4871 return I->getOpcode() == UIToFP;
4872 }
4873 static bool classof(const Value *V) {
4874 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4875 }
4876};
4877
4878//===----------------------------------------------------------------------===//
4879// SIToFPInst Class
4880//===----------------------------------------------------------------------===//
4881
4882/// This class represents a cast from signed integer to floating point.
4883class SIToFPInst : public CastInst {
4884protected:
4885 // Note: Instruction needs to be a friend here to call cloneImpl.
4886 friend class Instruction;
4887
4888 /// Clone an identical SIToFPInst
4889 SIToFPInst *cloneImpl() const;
4890
4891public:
4892 /// Constructor with insert-before-instruction semantics
4893 SIToFPInst(
4894 Value *S, ///< The value to be converted
4895 Type *Ty, ///< The type to convert to
4896 const Twine &NameStr = "", ///< A name for the new instruction
4897 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4898 );
4899
4900 /// Constructor with insert-at-end-of-block semantics
4901 SIToFPInst(
4902 Value *S, ///< The value to be converted
4903 Type *Ty, ///< The type to convert to
4904 const Twine &NameStr, ///< A name for the new instruction
4905 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4906 );
4907
4908 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4909 static bool classof(const Instruction *I) {
4910 return I->getOpcode() == SIToFP;
4911 }
4912 static bool classof(const Value *V) {
4913 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4914 }
4915};
4916
4917//===----------------------------------------------------------------------===//
4918// FPToUIInst Class
4919//===----------------------------------------------------------------------===//
4920
4921/// This class represents a cast from floating point to unsigned integer
4922class FPToUIInst : public CastInst {
4923protected:
4924 // Note: Instruction needs to be a friend here to call cloneImpl.
4925 friend class Instruction;
4926
4927 /// Clone an identical FPToUIInst
4928 FPToUIInst *cloneImpl() const;
4929
4930public:
4931 /// Constructor with insert-before-instruction semantics
4932 FPToUIInst(
4933 Value *S, ///< The value to be converted
4934 Type *Ty, ///< The type to convert to
4935 const Twine &NameStr = "", ///< A name for the new instruction
4936 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4937 );
4938
4939 /// Constructor with insert-at-end-of-block semantics
4940 FPToUIInst(
4941 Value *S, ///< The value to be converted
4942 Type *Ty, ///< The type to convert to
4943 const Twine &NameStr, ///< A name for the new instruction
4944 BasicBlock *InsertAtEnd ///< Where to insert the new instruction
4945 );
4946
4947 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4948 static bool classof(const Instruction *I) {
4949 return I->getOpcode() == FPToUI;
4950 }
4951 static bool classof(const Value *V) {
4952 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4953 }
4954};
4955
4956//===----------------------------------------------------------------------===//
4957// FPToSIInst Class
4958//===----------------------------------------------------------------------===//
4959
4960/// This class represents a cast from floating point to signed integer.
4961class FPToSIInst : public CastInst {
4962protected:
4963 // Note: Instruction needs to be a friend here to call cloneImpl.
4964 friend class Instruction;
4965
4966 /// Clone an identical FPToSIInst
4967 FPToSIInst *cloneImpl() const;
4968
4969public:
4970 /// Constructor with insert-before-instruction semantics
4971 FPToSIInst(
4972 Value *S, ///< The value to be converted
4973 Type *Ty, ///< The type to convert to
4974 const Twine &NameStr = "", ///< A name for the new instruction
4975 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
4976 );
4977
4978 /// Constructor with insert-at-end-of-block semantics
4979 FPToSIInst(
4980 Value *S, ///< The value to be converted
4981 Type *Ty, ///< The type to convert to
4982 const Twine &NameStr, ///< A name for the new instruction
4983 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
4984 );
4985
4986 /// Methods for support type inquiry through isa, cast, and dyn_cast:
4987 static bool classof(const Instruction *I) {
4988 return I->getOpcode() == FPToSI;
4989 }
4990 static bool classof(const Value *V) {
4991 return isa<Instruction>(V) && classof(cast<Instruction>(V));
4992 }
4993};
4994
4995//===----------------------------------------------------------------------===//
4996// IntToPtrInst Class
4997//===----------------------------------------------------------------------===//
4998
4999/// This class represents a cast from an integer to a pointer.
5000class IntToPtrInst : public CastInst {
5001public:
5002 // Note: Instruction needs to be a friend here to call cloneImpl.
5003 friend class Instruction;
5004
5005 /// Constructor with insert-before-instruction semantics
5006 IntToPtrInst(
5007 Value *S, ///< The value to be converted
5008 Type *Ty, ///< The type to convert to
5009 const Twine &NameStr = "", ///< A name for the new instruction
5010 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5011 );
5012
5013 /// Constructor with insert-at-end-of-block semantics
5014 IntToPtrInst(
5015 Value *S, ///< The value to be converted
5016 Type *Ty, ///< The type to convert to
5017 const Twine &NameStr, ///< A name for the new instruction
5018 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5019 );
5020
5021 /// Clone an identical IntToPtrInst.
5022 IntToPtrInst *cloneImpl() const;
5023
5024 /// Returns the address space of this instruction's pointer type.
5025 unsigned getAddressSpace() const {
5026 return getType()->getPointerAddressSpace();
5027 }
5028
5029 // Methods for support type inquiry through isa, cast, and dyn_cast:
5030 static bool classof(const Instruction *I) {
5031 return I->getOpcode() == IntToPtr;
5032 }
5033 static bool classof(const Value *V) {
5034 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5035 }
5036};
5037
5038//===----------------------------------------------------------------------===//
5039// PtrToIntInst Class
5040//===----------------------------------------------------------------------===//
5041
5042/// This class represents a cast from a pointer to an integer.
5043class PtrToIntInst : public CastInst {
5044protected:
5045 // Note: Instruction needs to be a friend here to call cloneImpl.
5046 friend class Instruction;
5047
5048 /// Clone an identical PtrToIntInst.
5049 PtrToIntInst *cloneImpl() const;
5050
5051public:
5052 /// Constructor with insert-before-instruction semantics
5053 PtrToIntInst(
5054 Value *S, ///< The value to be converted
5055 Type *Ty, ///< The type to convert to
5056 const Twine &NameStr = "", ///< A name for the new instruction
5057 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5058 );
5059
5060 /// Constructor with insert-at-end-of-block semantics
5061 PtrToIntInst(
5062 Value *S, ///< The value to be converted
5063 Type *Ty, ///< The type to convert to
5064 const Twine &NameStr, ///< A name for the new instruction
5065 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5066 );
5067
5068 /// Gets the pointer operand.
5069 Value *getPointerOperand() { return getOperand(0); }
5070 /// Gets the pointer operand.
5071 const Value *getPointerOperand() const { return getOperand(0); }
5072 /// Gets the operand index of the pointer operand.
5073 static unsigned getPointerOperandIndex() { return 0U; }
5074
5075 /// Returns the address space of the pointer operand.
5076 unsigned getPointerAddressSpace() const {
5077 return getPointerOperand()->getType()->getPointerAddressSpace();
5078 }
5079
5080 // Methods for support type inquiry through isa, cast, and dyn_cast:
5081 static bool classof(const Instruction *I) {
5082 return I->getOpcode() == PtrToInt;
5083 }
5084 static bool classof(const Value *V) {
5085 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5086 }
5087};
5088
5089//===----------------------------------------------------------------------===//
5090// BitCastInst Class
5091//===----------------------------------------------------------------------===//
5092
5093/// This class represents a no-op cast from one type to another.
5094class BitCastInst : public CastInst {
5095protected:
5096 // Note: Instruction needs to be a friend here to call cloneImpl.
5097 friend class Instruction;
5098
5099 /// Clone an identical BitCastInst.
5100 BitCastInst *cloneImpl() const;
5101
5102public:
5103 /// Constructor with insert-before-instruction semantics
5104 BitCastInst(
5105 Value *S, ///< The value to be casted
5106 Type *Ty, ///< The type to casted to
5107 const Twine &NameStr = "", ///< A name for the new instruction
5108 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5109 );
5110
5111 /// Constructor with insert-at-end-of-block semantics
5112 BitCastInst(
5113 Value *S, ///< The value to be casted
5114 Type *Ty, ///< The type to casted to
5115 const Twine &NameStr, ///< A name for the new instruction
5116 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5117 );
5118
5119 // Methods for support type inquiry through isa, cast, and dyn_cast:
5120 static bool classof(const Instruction *I) {
5121 return I->getOpcode() == BitCast;
5122 }
5123 static bool classof(const Value *V) {
5124 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5125 }
5126};
5127
5128//===----------------------------------------------------------------------===//
5129// AddrSpaceCastInst Class
5130//===----------------------------------------------------------------------===//
5131
5132/// This class represents a conversion between pointers from one address space
5133/// to another.
5134class AddrSpaceCastInst : public CastInst {
5135protected:
5136 // Note: Instruction needs to be a friend here to call cloneImpl.
5137 friend class Instruction;
5138
5139 /// Clone an identical AddrSpaceCastInst.
5140 AddrSpaceCastInst *cloneImpl() const;
5141
5142public:
5143 /// Constructor with insert-before-instruction semantics
5144 AddrSpaceCastInst(
5145 Value *S, ///< The value to be casted
5146 Type *Ty, ///< The type to casted to
5147 const Twine &NameStr = "", ///< A name for the new instruction
5148 Instruction *InsertBefore = nullptr ///< Where to insert the new instruction
5149 );
5150
5151 /// Constructor with insert-at-end-of-block semantics
5152 AddrSpaceCastInst(
5153 Value *S, ///< The value to be casted
5154 Type *Ty, ///< The type to casted to
5155 const Twine &NameStr, ///< A name for the new instruction
5156 BasicBlock *InsertAtEnd ///< The block to insert the instruction into
5157 );
5158
5159 // Methods for support type inquiry through isa, cast, and dyn_cast:
5160 static bool classof(const Instruction *I) {
5161 return I->getOpcode() == AddrSpaceCast;
5162 }
5163 static bool classof(const Value *V) {
5164 return isa<Instruction>(V) && classof(cast<Instruction>(V));
5165 }
5166
5167 /// Gets the pointer operand.
5168 Value *getPointerOperand() {
5169 return getOperand(0);
5170 }
5171
5172 /// Gets the pointer operand.
5173 const Value *getPointerOperand() const {
5174 return getOperand(0);
5175 }
5176
5177 /// Gets the operand index of the pointer operand.
5178 static unsigned getPointerOperandIndex() {
5179 return 0U;
5180 }
5181
5182 /// Returns the address space of the pointer operand.
5183 unsigned getSrcAddressSpace() const {
5184 return getPointerOperand()->getType()->getPointerAddressSpace();
5185 }
5186
5187 /// Returns the address space of the result.
5188 unsigned getDestAddressSpace() const {
5189 return getType()->getPointerAddressSpace();
5190 }
5191};
5192
5193} // end namespace llvm
5194
5195#endif // LLVM_IR_INSTRUCTIONS_H
←

/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instruction.h

1//===-- llvm/Instruction.h - Instruction class definition -------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains the declaration of the Instruction class, which is the
11// base class for all of the LLVM instructions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRUCTION_H
16#define LLVM_IR_INSTRUCTION_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/ADT/ilist_node.h"
22#include "llvm/IR/DebugLoc.h"
23#include "llvm/IR/SymbolTableListTraits.h"
24#include "llvm/IR/User.h"
25#include "llvm/IR/Value.h"
26#include "llvm/Support/Casting.h"
27#include <algorithm>
28#include <cassert>
29#include <cstdint>
30#include <utility>
31
32namespace llvm {
33
34class BasicBlock;
35class FastMathFlags;
36class MDNode;
37class Module;
38struct AAMDNodes;
39
40template <> struct ilist_alloc_traits<Instruction> {
41 static inline void deleteNode(Instruction *V);
42};
43
44class Instruction : public User,
45 public ilist_node_with_parent<Instruction, BasicBlock> {
46 BasicBlock *Parent;
47 DebugLoc DbgLoc; // 'dbg' Metadata cache.
48
49 enum {
50 /// This is a bit stored in the SubClassData field which indicates whether
51 /// this instruction has metadata attached to it or not.
52 HasMetadataBit = 1 << 15
53 };
54
55protected:
56 ~Instruction(); // Use deleteValue() to delete a generic Instruction.
57
58public:
59 Instruction(const Instruction &) = delete;
60 Instruction &operator=(const Instruction &) = delete;
61
62 /// Specialize the methods defined in Value, as we know that an instruction
63 /// can only be used by other instructions.
64 Instruction *user_back() { return cast<Instruction>(*user_begin());}
65 const Instruction *user_back() const { return cast<Instruction>(*user_begin());}
66
67 inline const BasicBlock *getParent() const { return Parent; }
68 inline BasicBlock *getParent() { return Parent; }
69
70 /// Return the module owning the function this instruction belongs to
71 /// or nullptr it the function does not have a module.
72 ///
73 /// Note: this is undefined behavior if the instruction does not have a
74 /// parent, or the parent basic block does not have a parent function.
75 const Module *getModule() const;
76 Module *getModule() {
77 return const_cast<Module *>(
78 static_cast<const Instruction *>(this)->getModule());
79 }
80
81 /// Return the function this instruction belongs to.
82 ///
83 /// Note: it is undefined behavior to call this on an instruction not
84 /// currently inserted into a function.
85 const Function *getFunction() const;
86 Function *getFunction() {
87 return const_cast<Function *>(
88 static_cast<const Instruction *>(this)->getFunction());
89 }
90
91 /// This method unlinks 'this' from the containing basic block, but does not
92 /// delete it.
93 void removeFromParent();
94
95 /// This method unlinks 'this' from the containing basic block and deletes it.
96 ///
97 /// \returns an iterator pointing to the element after the erased one
98 SymbolTableList<Instruction>::iterator eraseFromParent();
99
100 /// Insert an unlinked instruction into a basic block immediately before
101 /// the specified instruction.
102 void insertBefore(Instruction *InsertPos);
103
104 /// Insert an unlinked instruction into a basic block immediately after the
105 /// specified instruction.
106 void insertAfter(Instruction *InsertPos);
107
108 /// Unlink this instruction from its current basic block and insert it into
109 /// the basic block that MovePos lives in, right before MovePos.
110 void moveBefore(Instruction *MovePos);
111
112 /// Unlink this instruction and insert into BB before I.
113 ///
114 /// \pre I is a valid iterator into BB.
115 void moveBefore(BasicBlock &BB, SymbolTableList<Instruction>::iterator I);
116
117 /// Unlink this instruction from its current basic block and insert it into
118 /// the basic block that MovePos lives in, right after MovePos.
119 void moveAfter(Instruction *MovePos);
120
121 //===--------------------------------------------------------------------===//
122 // Subclass classification.
123 //===--------------------------------------------------------------------===//
124
125 /// Returns a member of one of the enums like Instruction::Add.
126 unsigned getOpcode() const { return getValueID() - InstructionVal; }
21
←
Calling 'Value::getValueID'
→
22
←
Returning from 'Value::getValueID'
→
127
128 const char *getOpcodeName() const { return getOpcodeName(getOpcode()); }
129 bool isTerminator() const { return isTerminator(getOpcode()); }
130 bool isBinaryOp() const { return isBinaryOp(getOpcode()); }
131 bool isShift() { return isShift(getOpcode()); }
132 bool isCast() const { return isCast(getOpcode()); }
133 bool isFuncletPad() const { return isFuncletPad(getOpcode()); }
134
135 static const char* getOpcodeName(unsigned OpCode);
136
137 static inline bool isTerminator(unsigned OpCode) {
138 return OpCode >= TermOpsBegin && OpCode < TermOpsEnd;
139 }
140
141 static inline bool isBinaryOp(unsigned Opcode) {
142 return Opcode >= BinaryOpsBegin && Opcode < BinaryOpsEnd;
143 }
144
145 /// Determine if the Opcode is one of the shift instructions.
146 static inline bool isShift(unsigned Opcode) {
147 return Opcode >= Shl && Opcode <= AShr;
148 }
149
150 /// Return true if this is a logical shift left or a logical shift right.
151 inline bool isLogicalShift() const {
152 return getOpcode() == Shl || getOpcode() == LShr;
153 }
154
155 /// Return true if this is an arithmetic shift right.
156 inline bool isArithmeticShift() const {
157 return getOpcode() == AShr;
158 }
159
160 /// Determine if the Opcode is and/or/xor.
161 static inline bool isBitwiseLogicOp(unsigned Opcode) {
162 return Opcode == And || Opcode == Or || Opcode == Xor;
163 }
164
165 /// Return true if this is and/or/xor.
166 inline bool isBitwiseLogicOp() const {
167 return isBitwiseLogicOp(getOpcode());
168 }
169
170 /// Determine if the OpCode is one of the CastInst instructions.
171 static inline bool isCast(unsigned OpCode) {
172 return OpCode >= CastOpsBegin && OpCode < CastOpsEnd;
173 }
174
175 /// Determine if the OpCode is one of the FuncletPadInst instructions.
176 static inline bool isFuncletPad(unsigned OpCode) {
177 return OpCode >= FuncletPadOpsBegin && OpCode < FuncletPadOpsEnd;
178 }
179
180 //===--------------------------------------------------------------------===//
181 // Metadata manipulation.
182 //===--------------------------------------------------------------------===//
183
184 /// Return true if this instruction has any metadata attached to it.
185 bool hasMetadata() const { return DbgLoc || hasMetadataHashEntry(); }
186
187 /// Return true if this instruction has metadata attached to it other than a
188 /// debug location.
189 bool hasMetadataOtherThanDebugLoc() const {
190 return hasMetadataHashEntry();
191 }
192
193 /// Get the metadata of given kind attached to this Instruction.
194 /// If the metadata is not found then return null.
195 MDNode *getMetadata(unsigned KindID) const {
196 if (!hasMetadata()) return nullptr;
197 return getMetadataImpl(KindID);
198 }
199
200 /// Get the metadata of given kind attached to this Instruction.
201 /// If the metadata is not found then return null.
202 MDNode *getMetadata(StringRef Kind) const {
203 if (!hasMetadata()) return nullptr;
204 return getMetadataImpl(Kind);
205 }
206
207 /// Get all metadata attached to this Instruction. The first element of each
208 /// pair returned is the KindID, the second element is the metadata value.
209 /// This list is returned sorted by the KindID.
210 void
211 getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
212 if (hasMetadata())
213 getAllMetadataImpl(MDs);
214 }
215
216 /// This does the same thing as getAllMetadata, except that it filters out the
217 /// debug location.
218 void getAllMetadataOtherThanDebugLoc(
219 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
220 if (hasMetadataOtherThanDebugLoc())
221 getAllMetadataOtherThanDebugLocImpl(MDs);
222 }
223
224 /// Fills the AAMDNodes structure with AA metadata from this instruction.
225 /// When Merge is true, the existing AA metadata is merged with that from this
226 /// instruction providing the most-general result.
227 void getAAMetadata(AAMDNodes &N, bool Merge = false) const;
228
229 /// Set the metadata of the specified kind to the specified node. This updates
230 /// or replaces metadata if already present, or removes it if Node is null.
231 void setMetadata(unsigned KindID, MDNode *Node);
232 void setMetadata(StringRef Kind, MDNode *Node);
233
234 /// Copy metadata from \p SrcInst to this instruction. \p WL, if not empty,
235 /// specifies the list of meta data that needs to be copied. If \p WL is
236 /// empty, all meta data will be copied.
237 void copyMetadata(const Instruction &SrcInst,
238 ArrayRef<unsigned> WL = ArrayRef<unsigned>());
239
240 /// If the instruction has "branch_weights" MD_prof metadata and the MDNode
241 /// has three operands (including name string), swap the order of the
242 /// metadata.
243 void swapProfMetadata();
244
245 /// Drop all unknown metadata except for debug locations.
246 /// @{
247 /// Passes are required to drop metadata they don't understand. This is a
248 /// convenience method for passes to do so.
249 void dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs);
250 void dropUnknownNonDebugMetadata() {
251 return dropUnknownNonDebugMetadata(None);
252 }
253 void dropUnknownNonDebugMetadata(unsigned ID1) {
254 return dropUnknownNonDebugMetadata(makeArrayRef(ID1));
255 }
256 void dropUnknownNonDebugMetadata(unsigned ID1, unsigned ID2) {
257 unsigned IDs[] = {ID1, ID2};
258 return dropUnknownNonDebugMetadata(IDs);
259 }
260 /// @}
261
262 /// Sets the metadata on this instruction from the AAMDNodes structure.
263 void setAAMetadata(const AAMDNodes &N);
264
265 /// Retrieve the raw weight values of a conditional branch or select.
266 /// Returns true on success with profile weights filled in.
267 /// Returns false if no metadata or invalid metadata was found.
268 bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) const;
269
270 /// Retrieve total raw weight values of a branch.
271 /// Returns true on success with profile total weights filled in.
272 /// Returns false if no metadata was found.
273 bool extractProfTotalWeight(uint64_t &TotalVal) const;
274
275 /// Updates branch_weights metadata by scaling it by \p S / \p T.
276 void updateProfWeight(uint64_t S, uint64_t T);
277
278 /// Sets the branch_weights metadata to \p W for CallInst.
279 void setProfWeight(uint64_t W);
280
281 /// Set the debug location information for this instruction.
282 void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); }
283
284 /// Return the debug location for this node as a DebugLoc.
285 const DebugLoc &getDebugLoc() const { return DbgLoc; }
286
287 /// Set or clear the nsw flag on this instruction, which must be an operator
288 /// which supports this flag. See LangRef.html for the meaning of this flag.
289 void setHasNoUnsignedWrap(bool b = true);
290
291 /// Set or clear the nsw flag on this instruction, which must be an operator
292 /// which supports this flag. See LangRef.html for the meaning of this flag.
293 void setHasNoSignedWrap(bool b = true);
294
295 /// Set or clear the exact flag on this instruction, which must be an operator
296 /// which supports this flag. See LangRef.html for the meaning of this flag.
297 void setIsExact(bool b = true);
298
299 /// Determine whether the no unsigned wrap flag is set.
300 bool hasNoUnsignedWrap() const;
301
302 /// Determine whether the no signed wrap flag is set.
303 bool hasNoSignedWrap() const;
304
305 /// Drops flags that may cause this instruction to evaluate to poison despite
306 /// having non-poison inputs.
307 void dropPoisonGeneratingFlags();
308
309 /// Determine whether the exact flag is set.
310 bool isExact() const;
311
312 /// Set or clear all fast-math-flags on this instruction, which must be an
313 /// operator which supports this flag. See LangRef.html for the meaning of
314 /// this flag.
315 void setFast(bool B);
316
317 /// Set or clear the reassociation flag on this instruction, which must be
318 /// an operator which supports this flag. See LangRef.html for the meaning of
319 /// this flag.
320 void setHasAllowReassoc(bool B);
321
322 /// Set or clear the no-nans flag on this instruction, which must be an
323 /// operator which supports this flag. See LangRef.html for the meaning of
324 /// this flag.
325 void setHasNoNaNs(bool B);
326
327 /// Set or clear the no-infs flag on this instruction, which must be an
328 /// operator which supports this flag. See LangRef.html for the meaning of
329 /// this flag.
330 void setHasNoInfs(bool B);
331
332 /// Set or clear the no-signed-zeros flag on this instruction, which must be
333 /// an operator which supports this flag. See LangRef.html for the meaning of
334 /// this flag.
335 void setHasNoSignedZeros(bool B);
336
337 /// Set or clear the allow-reciprocal flag on this instruction, which must be
338 /// an operator which supports this flag. See LangRef.html for the meaning of
339 /// this flag.
340 void setHasAllowReciprocal(bool B);
341
342 /// Set or clear the approximate-math-functions flag on this instruction,
343 /// which must be an operator which supports this flag. See LangRef.html for
344 /// the meaning of this flag.
345 void setHasApproxFunc(bool B);
346
347 /// Convenience function for setting multiple fast-math flags on this
348 /// instruction, which must be an operator which supports these flags. See
349 /// LangRef.html for the meaning of these flags.
350 void setFastMathFlags(FastMathFlags FMF);
351
352 /// Convenience function for transferring all fast-math flag values to this
353 /// instruction, which must be an operator which supports these flags. See
354 /// LangRef.html for the meaning of these flags.
355 void copyFastMathFlags(FastMathFlags FMF);
356
357 /// Determine whether all fast-math-flags are set.
358 bool isFast() const;
359
360 /// Determine whether the allow-reassociation flag is set.
361 bool hasAllowReassoc() const;
362
363 /// Determine whether the no-NaNs flag is set.
364 bool hasNoNaNs() const;
365
366 /// Determine whether the no-infs flag is set.
367 bool hasNoInfs() const;
368
369 /// Determine whether the no-signed-zeros flag is set.
370 bool hasNoSignedZeros() const;
371
372 /// Determine whether the allow-reciprocal flag is set.
373 bool hasAllowReciprocal() const;
374
375 /// Determine whether the allow-contract flag is set.
376 bool hasAllowContract() const;
377
378 /// Determine whether the approximate-math-functions flag is set.
379 bool hasApproxFunc() const;
380
381 /// Convenience function for getting all the fast-math flags, which must be an
382 /// operator which supports these flags. See LangRef.html for the meaning of
383 /// these flags.
384 FastMathFlags getFastMathFlags() const;
385
386 /// Copy I's fast-math flags
387 void copyFastMathFlags(const Instruction *I);
388
389 /// Convenience method to copy supported exact, fast-math, and (optionally)
390 /// wrapping flags from V to this instruction.
391 void copyIRFlags(const Value *V, bool IncludeWrapFlags = true);
392
393 /// Logical 'and' of any supported wrapping, exact, and fast-math flags of
394 /// V and this instruction.
395 void andIRFlags(const Value *V);
396
397 /// Merge 2 debug locations and apply it to the Instruction. If the
398 /// instruction is a CallIns, we need to traverse the inline chain to find
399 /// the common scope. This is not efficient for N-way merging as each time
400 /// you merge 2 iterations, you need to rebuild the hashmap to find the
401 /// common scope. However, we still choose this API because:
402 /// 1) Simplicity: it takes 2 locations instead of a list of locations.
403 /// 2) In worst case, it increases the complexity from O(N*I) to
404 /// O(2*N*I), where N is # of Instructions to merge, and I is the
405 /// maximum level of inline stack. So it is still linear.
406 /// 3) Merging of call instructions should be extremely rare in real
407 /// applications, thus the N-way merging should be in code path.
408 /// The DebugLoc attached to this instruction will be overwritten by the
409 /// merged DebugLoc.
410 void applyMergedLocation(const DILocation *LocA, const DILocation *LocB);
411
412private:
413 /// Return true if we have an entry in the on-the-side metadata hash.
414 bool hasMetadataHashEntry() const {
415 return (getSubclassDataFromValue() & HasMetadataBit) != 0;
416 }
417
418 // These are all implemented in Metadata.cpp.
419 MDNode *getMetadataImpl(unsigned KindID) const;
420 MDNode *getMetadataImpl(StringRef Kind) const;
421 void
422 getAllMetadataImpl(SmallVectorImpl<std::pair<unsigned, MDNode *>> &) const;
423 void getAllMetadataOtherThanDebugLocImpl(
424 SmallVectorImpl<std::pair<unsigned, MDNode *>> &) const;
425 /// Clear all hashtable-based metadata from this instruction.
426 void clearMetadataHashEntries();
427
428public:
429 //===--------------------------------------------------------------------===//
430 // Predicates and helper methods.
431 //===--------------------------------------------------------------------===//
432
433 /// Return true if the instruction is associative:
434 ///
435 /// Associative operators satisfy: x op (y op z) === (x op y) op z
436 ///
437 /// In LLVM, the Add, Mul, And, Or, and Xor operators are associative.
438 ///
439 bool isAssociative() const LLVM_READONLY__attribute__((__pure__));
440 static bool isAssociative(unsigned Opcode) {
441 return Opcode == And || Opcode == Or || Opcode == Xor ||
442 Opcode == Add || Opcode == Mul;
443 }
444
445 /// Return true if the instruction is commutative:
446 ///
447 /// Commutative operators satisfy: (x op y) === (y op x)
448 ///
449 /// In LLVM, these are the commutative operators, plus SetEQ and SetNE, when
450 /// applied to any type.
451 ///
452 bool isCommutative() const { return isCommutative(getOpcode()); }
453 static bool isCommutative(unsigned Opcode) {
454 switch (Opcode) {
455 case Add: case FAdd:
456 case Mul: case FMul:
457 case And: case Or: case Xor:
458 return true;
459 default:
460 return false;
461 }
462 }
463
464 /// Return true if the instruction is idempotent:
465 ///
466 /// Idempotent operators satisfy: x op x === x
467 ///
468 /// In LLVM, the And and Or operators are idempotent.
469 ///
470 bool isIdempotent() const { return isIdempotent(getOpcode()); }
471 static bool isIdempotent(unsigned Opcode) {
472 return Opcode == And || Opcode == Or;
473 }
474
475 /// Return true if the instruction is nilpotent:
476 ///
477 /// Nilpotent operators satisfy: x op x === Id,
478 ///
479 /// where Id is the identity for the operator, i.e. a constant such that
480 /// x op Id === x and Id op x === x for all x.
481 ///
482 /// In LLVM, the Xor operator is nilpotent.
483 ///
484 bool isNilpotent() const { return isNilpotent(getOpcode()); }
485 static bool isNilpotent(unsigned Opcode) {
486 return Opcode == Xor;
487 }
488
489 /// Return true if this instruction may modify memory.
490 bool mayWriteToMemory() const;
491
492 /// Return true if this instruction may read memory.
493 bool mayReadFromMemory() const;
494
495 /// Return true if this instruction may read or write memory.
496 bool mayReadOrWriteMemory() const {
497 return mayReadFromMemory() || mayWriteToMemory();
498 }
499
500 /// Return true if this instruction has an AtomicOrdering of unordered or
501 /// higher.
502 bool isAtomic() const;
503
504 /// Return true if this atomic instruction loads from memory.
505 bool hasAtomicLoad() const;
506
507 /// Return true if this atomic instruction stores to memory.
508 bool hasAtomicStore() const;
509
510 /// Return true if this instruction may throw an exception.
511 bool mayThrow() const;
512
513 /// Return true if this instruction behaves like a memory fence: it can load
514 /// or store to memory location without being given a memory location.
515 bool isFenceLike() const {
516 switch (getOpcode()) {
517 default:
518 return false;
519 // This list should be kept in sync with the list in mayWriteToMemory for
520 // all opcodes which don't have a memory location.
521 case Instruction::Fence:
522 case Instruction::CatchPad:
523 case Instruction::CatchRet:
524 case Instruction::Call:
525 case Instruction::Invoke:
526 return true;
527 }
528 }
529
530 /// Return true if the instruction may have side effects.
531 ///
532 /// Note that this does not consider malloc and alloca to have side
533 /// effects because the newly allocated memory is completely invisible to
534 /// instructions which don't use the returned value. For cases where this
535 /// matters, isSafeToSpeculativelyExecute may be more appropriate.
536 bool mayHaveSideEffects() const { return mayWriteToMemory() || mayThrow(); }
537
538 /// Return true if the instruction is a variety of EH-block.
539 bool isEHPad() const {
540 switch (getOpcode()) {
541 case Instruction::CatchSwitch:
542 case Instruction::CatchPad:
543 case Instruction::CleanupPad:
544 case Instruction::LandingPad:
545 return true;
546 default:
547 return false;
548 }
549 }
550
551 /// Create a copy of 'this' instruction that is identical in all ways except
552 /// the following:
553 /// * The instruction has no parent
554 /// * The instruction has no name
555 ///
556 Instruction *clone() const;
557
558 /// Return true if the specified instruction is exactly identical to the
559 /// current one. This means that all operands match and any extra information
560 /// (e.g. load is volatile) agree.
561 bool isIdenticalTo(const Instruction *I) const;
562
563 /// This is like isIdenticalTo, except that it ignores the
564 /// SubclassOptionalData flags, which may specify conditions under which the
565 /// instruction's result is undefined.
566 bool isIdenticalToWhenDefined(const Instruction *I) const;
567
568 /// When checking for operation equivalence (using isSameOperationAs) it is
569 /// sometimes useful to ignore certain attributes.
570 enum OperationEquivalenceFlags {
571 /// Check for equivalence ignoring load/store alignment.
572 CompareIgnoringAlignment = 1<<0,
573 /// Check for equivalence treating a type and a vector of that type
574 /// as equivalent.
575 CompareUsingScalarTypes = 1<<1
576 };
577
578 /// This function determines if the specified instruction executes the same
579 /// operation as the current one. This means that the opcodes, type, operand
580 /// types and any other factors affecting the operation must be the same. This
581 /// is similar to isIdenticalTo except the operands themselves don't have to
582 /// be identical.
583 /// @returns true if the specified instruction is the same operation as
584 /// the current one.
585 /// @brief Determine if one instruction is the same operation as another.
586 bool isSameOperationAs(const Instruction *I, unsigned flags = 0) const;
587
588 /// Return true if there are any uses of this instruction in blocks other than
589 /// the specified block. Note that PHI nodes are considered to evaluate their
590 /// operands in the corresponding predecessor block.
591 bool isUsedOutsideOfBlock(const BasicBlock *BB) const;
592
593
594 /// Methods for support type inquiry through isa, cast, and dyn_cast:
595 static bool classof(const Value *V) {
596 return V->getValueID() >= Value::InstructionVal;
597 }
598
599 //----------------------------------------------------------------------
600 // Exported enumerations.
601 //
602 enum TermOps { // These terminate basic blocks
603#define FIRST_TERM_INST(N) TermOpsBegin = N,
604#define HANDLE_TERM_INST(N, OPC, CLASS) OPC = N,
605#define LAST_TERM_INST(N) TermOpsEnd = N+1
606#include "llvm/IR/Instruction.def"
607 };
608
609 enum BinaryOps {
610#define FIRST_BINARY_INST(N) BinaryOpsBegin = N,
611#define HANDLE_BINARY_INST(N, OPC, CLASS) OPC = N,
612#define LAST_BINARY_INST(N) BinaryOpsEnd = N+1
613#include "llvm/IR/Instruction.def"
614 };
615
616 enum MemoryOps {
617#define FIRST_MEMORY_INST(N) MemoryOpsBegin = N,
618#define HANDLE_MEMORY_INST(N, OPC, CLASS) OPC = N,
619#define LAST_MEMORY_INST(N) MemoryOpsEnd = N+1
620#include "llvm/IR/Instruction.def"
621 };
622
623 enum CastOps {
624#define FIRST_CAST_INST(N) CastOpsBegin = N,
625#define HANDLE_CAST_INST(N, OPC, CLASS) OPC = N,
626#define LAST_CAST_INST(N) CastOpsEnd = N+1
627#include "llvm/IR/Instruction.def"
628 };
629
630 enum FuncletPadOps {
631#define FIRST_FUNCLETPAD_INST(N) FuncletPadOpsBegin = N,
632#define HANDLE_FUNCLETPAD_INST(N, OPC, CLASS) OPC = N,
633#define LAST_FUNCLETPAD_INST(N) FuncletPadOpsEnd = N+1
634#include "llvm/IR/Instruction.def"
635 };
636
637 enum OtherOps {
638#define FIRST_OTHER_INST(N) OtherOpsBegin = N,
639#define HANDLE_OTHER_INST(N, OPC, CLASS) OPC = N,
640#define LAST_OTHER_INST(N) OtherOpsEnd = N+1
641#include "llvm/IR/Instruction.def"
642 };
643
644private:
645 friend class SymbolTableListTraits<Instruction>;
646
647 // Shadow Value::setValueSubclassData with a private forwarding method so that
648 // subclasses cannot accidentally use it.
649 void setValueSubclassData(unsigned short D) {
650 Value::setValueSubclassData(D);
651 }
652
653 unsigned short getSubclassDataFromValue() const {
654 return Value::getSubclassDataFromValue();
655 }
656
657 void setHasMetadataHashEntry(bool V) {
658 setValueSubclassData((getSubclassDataFromValue() & ~HasMetadataBit) |
659 (V ? HasMetadataBit : 0));
660 }
661
662 void setParent(BasicBlock *P);
663
664protected:
665 // Instruction subclasses can stick up to 15 bits of stuff into the
666 // SubclassData field of instruction with these members.
667
668 // Verify that only the low 15 bits are used.
669 void setInstructionSubclassData(unsigned short D) {
670 assert((D & HasMetadataBit) == 0 && "Out of range value put into field")(static_cast <bool> ((D & HasMetadataBit) == 0 &&
"Out of range value put into field") ? void (0) : __assert_fail
("(D & HasMetadataBit) == 0 && \"Out of range value put into field\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/include/llvm/IR/Instruction.h"
, 670, __extension__ __PRETTY_FUNCTION__));
671 setValueSubclassData((getSubclassDataFromValue() & HasMetadataBit) | D);
672 }
673
674 unsigned getSubclassDataFromInstruction() const {
675 return getSubclassDataFromValue() & ~HasMetadataBit;
676 }
677
678 Instruction(Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
679 Instruction *InsertBefore = nullptr);
680 Instruction(Type *Ty, unsigned iType, Use *Ops, unsigned NumOps,
681 BasicBlock *InsertAtEnd);
682
683private:
684 /// Create a copy of this instruction.
685 Instruction *cloneImpl() const;
686};
687
688inline void ilist_alloc_traits<Instruction>::deleteNode(Instruction *V) {
689 V->deleteValue();
690}
691
692} // end namespace llvm
693
694#endif // LLVM_IR_INSTRUCTION_H