22 #define LV_NAME "loop-vectorize" 23 #define DEBUG_TYPE LV_NAME 29 cl::desc(
"Enable if-conversion during vectorization."));
33 cl::desc(
"The maximum allowed number of runtime memory checks with a " 34 "vectorize(enable) pragma."));
38 cl::desc(
"The maximum number of SCEV checks allowed."));
42 cl::desc(
"The maximum number of SCEV checks allowed with a " 43 "vectorize(enable) pragma"));
66 R <<
"loop not vectorized: ";
70 bool LoopVectorizeHints::Hint::validate(
unsigned Val) {
79 return (Val == 0 || Val == 1);
85 bool InterleaveOnlyWhenForced,
88 Interleave(
"interleave.count", InterleaveOnlyWhenForced, HK_UNROLL),
89 Force(
"vectorize.enable", FK_Undefined, HK_FORCE),
90 IsVectorized(
"isvectorized", 0, HK_ISVECTORIZED), TheLoop(L), ORE(ORE) {
92 getHintsFromMetadata();
98 if (IsVectorized.Value != 1)
102 IsVectorized.Value = Width.Value == 1 && Interleave.Value == 1;
103 LLVM_DEBUG(
if (InterleaveOnlyWhenForced && Interleave.Value == 1)
dbgs()
104 <<
"LV: Interleaving disabled by the pass manager\n");
117 {
Twine(Prefix(),
"vectorize.").
str(),
118 Twine(Prefix(),
"interleave.").
str()},
123 IsVectorized.Value = 1;
129 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: #pragma vectorize disable.\n");
135 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: No #pragma vectorize enable.\n");
141 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Disabled/already vectorized.\n");
149 <<
"loop not vectorized: vectorization and interleaving are " 150 "explicitly disabled, or the loop has already been " 165 TheLoop->getStartLoc(),
166 TheLoop->getHeader())
167 <<
"loop not vectorized: vectorization is explicitly disabled";
170 TheLoop->getStartLoc(), TheLoop->getHeader());
171 R <<
"loop not vectorized";
173 R <<
" (Force=" <<
NV(
"Force",
true);
174 if (Width.Value != 0)
175 R <<
", Vector Width=" <<
NV(
"VectorWidth", Width.Value);
176 if (Interleave.Value != 0)
177 R <<
", Interleave Count=" <<
NV(
"InterleaveCount", Interleave.Value);
195 void LoopVectorizeHints::getHintsFromMetadata() {
196 MDNode *LoopID = TheLoop->getLoopID();
204 for (
unsigned i = 1, ie = LoopID->
getNumOperands(); i < ie; ++i) {
211 if (!MD || MD->getNumOperands() == 0)
214 for (
unsigned i = 1, ie = MD->getNumOperands(); i < ie; ++i)
218 assert(Args.
size() == 0 &&
"too many arguments for MDString");
226 if (Args.
size() == 1)
227 setHint(Name, Args[0]);
241 Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized};
242 for (
auto H : Hints) {
243 if (Name ==
H->Name) {
244 if (
H->validate(Val))
247 LLVM_DEBUG(
dbgs() <<
"LV: ignoring invalid hint '" << Name <<
"'\n");
260 PassName,
"CantReorderFPOps", UnsafeAlgebraInst->getDebugLoc(),
261 UnsafeAlgebraInst->getParent())
262 <<
"loop not vectorized: cannot prove it is safe to reorder " 263 "floating-point operations";
269 bool PragmaThresholdReached =
271 bool ThresholdReached =
274 PragmaThresholdReached) {
279 <<
"loop not vectorized: cannot prove it is safe to reorder " 330 if (!LatchBr || LatchBr->isUnconditional()) {
339 dbgs() <<
"LV: Loop latch condition is not a compare instruction.\n");
343 Value *CondOp0 = LatchCmp->getOperand(0);
344 Value *CondOp1 = LatchCmp->getOperand(1);
345 Value *IVUpdate = IV->getIncomingValueForBlock(Latch);
348 LLVM_DEBUG(
dbgs() <<
"LV: Loop latch condition is not uniform.\n");
362 for (
Loop *SubLp : *Lp)
375 for (
Value *V : Phi.incoming_values())
376 if (
auto *
C = dyn_cast<Constant>(V))
409 if (!AllowedExit.
count(Inst))
415 LLVM_DEBUG(
dbgs() <<
"LV: Found an outside user for : " << *UI <<
'\n');
426 int Stride =
getPtrStride(PSE, Ptr, TheLoop, Strides,
true,
false);
427 if (Stride == 1 || Stride == -1)
433 return LAI->isUniform(V);
436 bool LoopVectorizationLegality::canVectorizeOuterLoop() {
437 assert(!TheLoop->empty() &&
"We are not vectorizing an outer loop.");
441 bool DoExtraAnalysis = ORE->allowExtraAnalysis(
DEBUG_TYPE);
448 LLVM_DEBUG(
dbgs() <<
"LV: Unsupported basic block terminator.\n");
449 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
450 <<
"loop control flow is not understood by vectorizer");
463 if (!EnableVPlanPredication && Br && Br->isConditional() &&
464 !TheLoop->isLoopInvariant(Br->getCondition()) &&
465 !LI->isLoopHeader(Br->getSuccessor(0)) &&
466 !LI->isLoopHeader(Br->getSuccessor(1))) {
468 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
469 <<
"loop control flow is not understood by vectorizer");
483 <<
"LV: Not vectorizing: Outer loop contains divergent loops.\n");
484 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
485 <<
"loop control flow is not understood by vectorizer");
493 if (!setupOuterLoopInductions()) {
495 dbgs() <<
"LV: Not vectorizing: Unsupported outer loop Phi(s).\n");
496 ORE->emit(createMissedAnalysis(
"UnsupportedPhi")
497 <<
"Unsupported outer loop Phi(s)");
507 void LoopVectorizationLegality::addInductionPhi(
510 Inductions[Phi] =
ID;
518 InductionCastsToIgnore.insert(*Casts.
begin());
541 if (!PrimaryInduction || PhiTy == WidestIndTy)
542 PrimaryInduction = Phi;
551 if (PSE.getUnionPredicate().isAlwaysTrue()) {
559 bool LoopVectorizationLegality::setupOuterLoopInductions() {
563 auto isSupportedPhi = [&](
PHINode &Phi) ->
bool {
567 addInductionPhi(&Phi, ID, AllowedExit);
574 <<
"LV: Found unsupported PHI for outer loop vectorization.\n");
585 bool LoopVectorizationLegality::canVectorizeInstrs() {
597 if (
auto *Phi = dyn_cast<PHINode>(&
I)) {
602 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood", Phi)
603 <<
"loop control flow is not understood by vectorizer");
622 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood", Phi)
623 <<
"control flow not understood by vectorizer");
634 Reductions[Phi] = RedDes;
654 addInductionPhi(Phi, ID, AllowedExit);
662 FirstOrderRecurrences.insert(Phi);
669 addInductionPhi(Phi, ID, AllowedExit);
673 ORE->emit(createMissedAnalysis(
"NonReductionValueUsedOutsideLoop", Phi)
674 <<
"value that could not be identified as " 675 "reduction is used outside the loop");
676 LLVM_DEBUG(
dbgs() <<
"LV: Found an unidentified PHI." << *Phi <<
"\n");
686 !isa<DbgInfoIntrinsic>(CI) &&
687 !(CI->getCalledFunction() && TLI &&
688 TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) {
693 TLI && CI->getCalledFunction() &&
694 CI->getType()->isFloatingPointTy() &&
695 TLI->getLibFunc(CI->getCalledFunction()->getName(), Func) &&
696 TLI->hasOptimizedCodeGen(Func);
703 ORE->emit(createMissedAnalysis(
"CantVectorizeLibcall", CI)
704 <<
"library call cannot be vectorized. " 705 "Try compiling with -fno-math-errno, -ffast-math, " 708 ORE->emit(createMissedAnalysis(
"CantVectorizeCall", CI)
709 <<
"call instruction cannot be vectorized");
712 dbgs() <<
"LV: Found a non-intrinsic callsite.\n");
720 auto *SE = PSE.getSE();
721 if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(1)), TheLoop)) {
722 ORE->emit(createMissedAnalysis(
"CantVectorizeIntrinsic", CI)
723 <<
"intrinsic instruction cannot be vectorized");
725 <<
"LV: Found unvectorizable intrinsic " << *CI <<
"\n");
733 !
I.getType()->isVoidTy()) ||
734 isa<ExtractElementInst>(
I)) {
735 ORE->emit(createMissedAnalysis(
"CantVectorizeInstructionReturnType", &
I)
736 <<
"instruction return type cannot be vectorized");
742 if (
auto *
ST = dyn_cast<StoreInst>(&
I)) {
743 Type *
T =
ST->getValueOperand()->getType();
745 ORE->emit(createMissedAnalysis(
"CantVectorizeStore",
ST)
746 <<
"store instruction cannot be vectorized");
755 }
else if (
I.getType()->isFloatingPointTy() && (CI ||
I.isBinaryOp()) &&
758 Hints->setPotentiallyUnsafe();
768 if (PSE.getUnionPredicate().isAlwaysTrue()) {
772 ORE->emit(createMissedAnalysis(
"ValueUsedOutsideLoop", &
I)
773 <<
"value cannot be used outside the loop");
779 if (!PrimaryInduction) {
780 LLVM_DEBUG(
dbgs() <<
"LV: Did not find one integer induction var.\n");
781 if (Inductions.empty()) {
782 ORE->emit(createMissedAnalysis(
"NoInductionVariable")
783 <<
"loop induction variable could not be identified");
785 }
else if (!WidestIndTy) {
786 ORE->emit(createMissedAnalysis(
"NoIntegerInductionVariable")
787 <<
"integer loop induction variable could not be identified");
795 if (PrimaryInduction && WidestIndTy != PrimaryInduction->getType())
796 PrimaryInduction =
nullptr;
801 bool LoopVectorizationLegality::canVectorizeMemory() {
802 LAI = &(*GetLAA)(*TheLoop);
807 "loop not vectorized: ", *LAR);
810 if (!LAI->canVectorizeMemory())
813 if (LAI->hasDependenceInvolvingLoopInvariantAddress()) {
814 ORE->emit(createMissedAnalysis(
"CantVectorizeStoreToLoopInvariantAddress")
815 <<
"write to a loop invariant address could not " 818 dbgs() <<
"LV: Non vectorizable stores to a uniform address\n");
821 Requirements->addRuntimePointerChecks(LAI->getNumRuntimePointerChecks());
822 PSE.addPredicate(LAI->getPSE().getUnionPredicate());
829 PHINode *PN = dyn_cast_or_null<PHINode>(In0);
833 return Inductions.count(PN);
838 return (Inst && InductionCastsToIgnore.count(Inst));
842 return isInductionPhi(V) || isCastedInductionVariable(V);
846 return FirstOrderRecurrences.count(Phi);
853 bool LoopVectorizationLegality::blockCanBePredicated(
855 const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
859 for (
Value *Operand :
I.operands()) {
860 if (
auto *
C = dyn_cast<Constant>(Operand))
865 if (
I.mayReadFromMemory()) {
869 if (!SafePtrs.
count(LI->getPointerOperand())) {
873 if (!IsAnnotatedParallel)
879 if (
I.mayWriteToMemory()) {
898 bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
900 ORE->emit(createMissedAnalysis(
"IfConversionDisabled")
901 <<
"if-conversion is disabled");
905 assert(TheLoop->getNumBlocks() > 1 &&
"Single block loops are vectorizable");
912 if (blockNeedsPredication(BB))
925 ORE->emit(createMissedAnalysis(
"LoopContainsSwitch", BB->
getTerminator())
926 <<
"loop contains a switch statement");
931 if (blockNeedsPredication(BB)) {
932 if (!blockCanBePredicated(BB, SafePointes)) {
933 ORE->emit(createMissedAnalysis(
"NoCFGForSelect", BB->
getTerminator())
934 <<
"control flow cannot be substituted for a select");
938 ORE->emit(createMissedAnalysis(
"NoCFGForSelect", BB->
getTerminator())
939 <<
"control flow cannot be substituted for a select");
949 bool LoopVectorizationLegality::canVectorizeLoopCFG(
Loop *Lp,
950 bool UseVPlanNativePath) {
952 "VPlan-native path is not enabled.");
962 bool DoExtraAnalysis = ORE->allowExtraAnalysis(
DEBUG_TYPE);
967 LLVM_DEBUG(
dbgs() <<
"LV: Loop doesn't have a legal pre-header.\n");
968 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
969 <<
"loop control flow is not understood by vectorizer");
978 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
979 <<
"loop control flow is not understood by vectorizer");
988 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
989 <<
"loop control flow is not understood by vectorizer");
1000 ORE->emit(createMissedAnalysis(
"CFGNotUnderstood")
1001 <<
"loop control flow is not understood by vectorizer");
1002 if (DoExtraAnalysis)
1011 bool LoopVectorizationLegality::canVectorizeLoopNestCFG(
1012 Loop *Lp,
bool UseVPlanNativePath) {
1016 bool DoExtraAnalysis = ORE->allowExtraAnalysis(
DEBUG_TYPE);
1017 if (!canVectorizeLoopCFG(Lp, UseVPlanNativePath)) {
1018 if (DoExtraAnalysis)
1026 for (
Loop *SubLp : *Lp)
1027 if (!canVectorizeLoopNestCFG(SubLp, UseVPlanNativePath)) {
1028 if (DoExtraAnalysis)
1042 bool DoExtraAnalysis = ORE->allowExtraAnalysis(
DEBUG_TYPE);
1045 if (!canVectorizeLoopNestCFG(TheLoop, UseVPlanNativePath)) {
1046 if (DoExtraAnalysis)
1053 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop: " << TheLoop->getHeader()->getName()
1058 if (!TheLoop->empty()) {
1059 assert(UseVPlanNativePath &&
"VPlan-native path is not enabled.");
1061 if (!canVectorizeOuterLoop()) {
1062 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Unsupported outer loop.\n");
1072 assert(TheLoop->empty() &&
"Inner loop expected.");
1074 unsigned NumBlocks = TheLoop->getNumBlocks();
1075 if (NumBlocks != 1 && !canVectorizeWithIfConvert()) {
1077 if (DoExtraAnalysis)
1084 if (!canVectorizeInstrs()) {
1085 LLVM_DEBUG(
dbgs() <<
"LV: Can't vectorize the instructions or CFG\n");
1086 if (DoExtraAnalysis)
1093 if (!canVectorizeMemory()) {
1094 LLVM_DEBUG(
dbgs() <<
"LV: Can't vectorize due to memory conflicts\n");
1095 if (DoExtraAnalysis)
1102 << (LAI->getRuntimePointerChecking()->Need
1103 ?
" (with a runtime bound check)" 1111 if (PSE.getUnionPredicate().getComplexity() > SCEVThreshold) {
1112 ORE->emit(createMissedAnalysis(
"TooManySCEVRunTimeChecks")
1113 <<
"Too many SCEV assumptions need to be made and checked " 1116 if (DoExtraAnalysis)
1131 LLVM_DEBUG(
dbgs() <<
"LV: checking if tail can be folded by masking.\n");
1133 if (!PrimaryInduction) {
1134 ORE->emit(createMissedAnalysis(
"NoPrimaryInduction")
1135 <<
"Missing a primary induction variable in the loop, which is " 1136 <<
"needed in order to fold tail by masking as required.");
1137 LLVM_DEBUG(
dbgs() <<
"LV: No primary induction, cannot fold tail by " 1143 if (!Reductions.empty()) {
1144 ORE->emit(createMissedAnalysis(
"ReductionFoldingTailByMasking")
1145 <<
"Cannot fold tail by masking in the presence of reductions.");
1146 LLVM_DEBUG(
dbgs() <<
"LV: Loop has reductions, cannot fold tail by " 1152 for (
auto *AE : AllowedExit) {
1156 if (TheLoop->contains(UI))
1158 ORE->emit(createMissedAnalysis(
"LiveOutFoldingTailByMasking")
1159 <<
"Cannot fold tail by masking in the presence of live outs.");
1160 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking, loop has an " 1161 <<
"outside user for : " << *UI <<
'\n');
1172 if (!blockCanBePredicated(BB, SafePointers)) {
1173 ORE->emit(createMissedAnalysis(
"NoCFGForSelect", BB->
getTerminator())
1174 <<
"control flow cannot be substituted for a select");
1175 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking as required.\n");
static bool isUniformLoop(Loop *Lp, Loop *OuterLp)
static unsigned RuntimeMemoryCheckThreshold
performing memory disambiguation checks at runtime do not make more than this number of comparisons...
A parsed version of the target data layout string in and methods for querying it. ...
This class is the base class for the comparison instructions.
unsigned getWidth() const
bool allowReordering() const
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
static Type * getWiderType(const DataLayout &DL, Type *Ty0, Type *Ty1)
DiagnosticInfoOptimizationBase::Argument NV
This class represents lattice values for constants.
Instruction * getUnsafeAlgebraInst()
Returns first unsafe algebra instruction in the PHI node's use-chain.
bool isCastedInductionVariable(const Value *V)
Returns True if V is a cast that is part of an induction def-use chain, and had been proven to be red...
static bool isInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, InductionDescriptor &D, const SCEV *Expr=nullptr, SmallVectorImpl< Instruction *> *CastsToIgnore=nullptr)
Returns true if Phi is an induction in the loop L.
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
static MDString * get(LLVMContext &Context, StringRef Str)
ConstantInt * getConstIntStepValue() const
LLVM_NODISCARD bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
void push_back(const T &Elt)
llvm::MDNode * makePostTransformationMetadata(llvm::LLVMContext &Context, MDNode *OrigLoopID, llvm::ArrayRef< llvm::StringRef > RemovePrefixes, llvm::ArrayRef< llvm::MDNode *> AddAttrs)
Create a new LoopID after the loop has been transformed.
bool isUniform(Value *V)
Returns true if the value V is uniform within the loop.
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class...
This class represents a function call, abstracting a target machine's calling convention.
int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp, const ValueToValueMap &StridesMap=ValueToValueMap(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of its element size.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
InductionKind getKind() const
const MDOperand & getOperand(unsigned I) const
An instruction for reading from memory.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
cl::opt< bool > EnableVPlanPredication
Value * getStartValue() const
LLVMContext & getContext() const
Get the context in which this basic block lives.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
bool hasUnsafeAlgebra()
Returns true if the recurrence has unsafe algebra which requires a relaxed floating-point model...
unsigned getNumBackEdges() const
Calculate the number of back edges to the loop header.
This file defines the LoopVectorizationLegality class.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static const unsigned MaxVectorWidth
Maximum SIMD width.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
static cl::opt< bool > EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden, cl::desc("Enable if-conversion during vectorization."))
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isIntegerTy() const
True if this is an instance of IntegerType.
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
BlockT * getHeader() const
int isConsecutivePtr(Value *Ptr)
Check if this pointer is consecutive when vectorizing.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Type * getType() const
All values are typed, get the type of this value.
bool isInductionVariable(const Value *V)
Returns True if V can be considered as an induction variable in this loop.
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
Value * getLoadStorePointerOperand(Value *V)
A helper function that returns the pointer operand of a load or store instruction.
An instruction for storing to memory.
static cl::opt< unsigned > PragmaVectorizeMemoryCheckThreshold("pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks with a " "vectorize(enable) pragma."))
bool blockNeedsPredication(BasicBlock *BB)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
StringRef getString() const
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
enum ForceKind getForce() const
initializer< Ty > init(const Ty &Val)
Integer induction variable. Step = C.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static bool hasOutsideLoopUser(const Loop *TheLoop, Instruction *Inst, SmallPtrSetImpl< Value *> &AllowedExit)
Check that the instruction has outside loop users and is not an identified reduction variable...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
Conditional or Unconditional Branch instruction.
Instruction * getUnsafeAlgebraInst()
Returns induction operator that does not have "fast-math" property and requires FP unsafe mode...
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
bool isPointerTy() const
True if this is an instance of PointerType.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
OptimizationRemarkAnalysis createLVMissedAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I=nullptr)
Create an analysis remark that explains why vectorization failed.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes, DemandedBits *DB=nullptr, AssumptionCache *AC=nullptr, DominatorTree *DT=nullptr)
Returns true if Phi is a reduction in TheLoop.
static bool isUniformLoopNest(Loop *Lp, Loop *OuterLp)
bool doesNotMeet(Function *F, Loop *L, const LoopVectorizeHints &Hints)
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
static bool isFirstOrderRecurrence(PHINode *Phi, Loop *TheLoop, DenseMap< Instruction *, Instruction *> &SinkAfter, DominatorTree *DT)
Returns true if Phi is a first-order recurrence.
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
unsigned getIsVectorized() const
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is the shared class of boolean and integer constants.
auto size(R &&Range, typename std::enable_if< std::is_same< typename std::iterator_traits< decltype(Range.begin())>::iterator_category, std::random_access_iterator_tag >::value, void >::type *=nullptr) -> decltype(std::distance(Range.begin(), Range.end()))
Get the size of a range.
A struct for saving information about induction variables.
bool canFoldTailByMasking()
Return true if we can vectorize this loop while folding its tail by masking.
testing::Matcher< const detail::ErrorHolder & > Failed()
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
DenseMap< const Value *, Value * > ValueToValueMap
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
bool isFirstOrderRecurrence(const PHINode *Phi)
Returns True if Phi is a first-order recurrence in this loop.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static const unsigned MaxInterleaveFactor
Maximum vectorization interleave count.
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the intrinsic has a scalar operand.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
PHINode * getCanonicalInductionVariable() const
Check to see if the loop has a canonical induction variable: an integer recurrence that starts at 0 a...
static bool canIfConvertPHINodes(BasicBlock *BB)
Check whether it is safe to if-convert this phi node.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
bool isInductionPhi(const Value *V)
Returns True if V is a Phi node of an induction variable in this loop.
Class for arbitrary precision integers.
iterator_range< user_iterator > users()
bool hasUnsafeAlgebra()
Returns true if the induction type is FP and the binary operator does not have the "fast-math" proper...
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced, OptimizationRemarkEmitter &ORE)
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
static IntegerType * getInt32Ty(LLVMContext &C)
LLVM_NODISCARD bool empty() const
Instruction * getLoopExitInstr()
StringRef getValueAsString() const
Return the attribute's value as a string.
Represents a single loop in the control flow graph.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Collection of parameters shared beetween the Loop Vectorizer and the Loop Access Analysis.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
std::string str() const
Return the twine contents as a std::string.
void setAlreadyVectorized()
Mark the loop L as already vectorized by setting the width to 1.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, DominatorTree *DT)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
static unsigned VectorizationInterleave
Interleave factor as overridden by the user.
static Type * convertPointerToIntegerType(const DataLayout &DL, Type *Ty)
StringRef - Represent a constant reference to a string, i.e.
static cl::opt< unsigned > VectorizeSCEVCheckThreshold("vectorize-scev-check-threshold", cl::init(16), cl::Hidden, cl::desc("The maximum number of SCEV checks allowed."))
static cl::opt< unsigned > PragmaVectorizeSCEVCheckThreshold("pragma-vectorize-scev-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum number of SCEV checks allowed with a " "vectorize(enable) pragma"))
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
static bool isInterleaveForced()
True if force-vector-interleave was specified by the user.
unsigned getNumOperands() const
Return number of MDNode operands.
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
void emitRemarkWithHints() const
Dumps all the hint information.
const BasicBlock * getParent() const