25 using namespace llvm::PatternMatch;
27 #define DEBUG_TYPE "loop-utils"
32 if (!Set.
count(dyn_cast<Instruction>(*
Use)))
38 Loop *TheLoop,
bool HasFunNoNaNAttr,
57 bool FoundReduxOp =
false;
63 bool FoundStartPHI =
false;
68 unsigned NumCmpSelectPatternInst = 0;
90 while (!Worklist.
empty()) {
100 bool IsAPhi = isa<PHINode>(Cur);
108 if (!Cur->
isCommutative() && !IsAPhi && !isa<SelectInst>(Cur) &&
109 !isa<ICmpInst>(Cur) && !isa<FCmpInst>(Cur) &&
114 ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);
119 if (!IsAPhi && Kind != RK_IntegerMinMax && Kind != RK_FloatMinMax &&
120 hasMultipleUsesOf(Cur, VisitedInsts))
124 if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
127 if (Kind == RK_IntegerMinMax &&
128 (isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
129 ++NumCmpSelectPatternInst;
130 if (Kind == RK_FloatMinMax && (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
131 ++NumCmpSelectPatternInst;
134 FoundReduxOp |= !IsAPhi;
151 if (ExitInstruction !=
nullptr || Cur == Phi)
160 ExitInstruction = Cur;
167 InstDesc IgnoredVal(
false,
nullptr);
168 if (VisitedInsts.
insert(UI).second) {
169 if (isa<PHINode>(UI))
173 }
else if (!isa<PHINode>(UI) &&
174 ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) &&
175 !isa<SelectInst>(UI)) ||
176 !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence()))
181 FoundStartPHI =
true;
189 if ((Kind == RK_IntegerMinMax || Kind == RK_FloatMinMax) &&
190 NumCmpSelectPatternInst != 2)
193 if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction)
216 assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) &&
217 "Expect a select instruction");
223 if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) {
230 if (!(Select = dyn_cast<SelectInst>(I)))
232 if (!(Cmp = dyn_cast<ICmpInst>(I->
getOperand(0))) &&
243 return InstDesc(Select, MRK_UIntMin);
245 return InstDesc(Select, MRK_UIntMax);
247 return InstDesc(Select, MRK_SIntMax);
249 return InstDesc(Select, MRK_SIntMin);
251 return InstDesc(Select, MRK_FloatMin);
253 return InstDesc(Select, MRK_FloatMax);
255 return InstDesc(Select, MRK_FloatMin);
257 return InstDesc(Select, MRK_FloatMax);
264 InstDesc &Prev,
bool HasFunNoNaNAttr) {
272 (Kind != RK_FloatMult && Kind != RK_FloatAdd && Kind != RK_FloatMinMax))
275 case Instruction::Sub:
276 case Instruction::Add:
277 return InstDesc(Kind == RK_IntegerAdd, I);
278 case Instruction::Mul:
279 return InstDesc(Kind == RK_IntegerMult, I);
281 return InstDesc(Kind == RK_IntegerAnd, I);
283 return InstDesc(Kind == RK_IntegerOr, I);
285 return InstDesc(Kind == RK_IntegerXor, I);
286 case Instruction::FMul:
287 return InstDesc(Kind == RK_FloatMult && FastMath, I);
288 case Instruction::FSub:
289 case Instruction::FAdd:
290 return InstDesc(Kind == RK_FloatAdd && FastMath, I);
291 case Instruction::FCmp:
292 case Instruction::ICmp:
294 if (Kind != RK_IntegerMinMax &&
295 (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))
297 return isMinMaxSelectCmpPattern(I, Prev);
303 unsigned NumUses = 0;
306 if (Insts.
count(dyn_cast<Instruction>(*
Use)))
317 bool HasFunNoNaNAttr =
false;
324 if (AddReductionVar(Phi, RK_IntegerAdd, TheLoop, HasFunNoNaNAttr, RedDes)) {
325 DEBUG(
dbgs() <<
"Found an ADD reduction PHI." << *Phi <<
"\n");
328 if (AddReductionVar(Phi, RK_IntegerMult, TheLoop, HasFunNoNaNAttr, RedDes)) {
329 DEBUG(
dbgs() <<
"Found a MUL reduction PHI." << *Phi <<
"\n");
332 if (AddReductionVar(Phi, RK_IntegerOr, TheLoop, HasFunNoNaNAttr, RedDes)) {
333 DEBUG(
dbgs() <<
"Found an OR reduction PHI." << *Phi <<
"\n");
336 if (AddReductionVar(Phi, RK_IntegerAnd, TheLoop, HasFunNoNaNAttr, RedDes)) {
337 DEBUG(
dbgs() <<
"Found an AND reduction PHI." << *Phi <<
"\n");
340 if (AddReductionVar(Phi, RK_IntegerXor, TheLoop, HasFunNoNaNAttr, RedDes)) {
341 DEBUG(
dbgs() <<
"Found a XOR reduction PHI." << *Phi <<
"\n");
344 if (AddReductionVar(Phi, RK_IntegerMinMax, TheLoop, HasFunNoNaNAttr,
346 DEBUG(
dbgs() <<
"Found a MINMAX reduction PHI." << *Phi <<
"\n");
349 if (AddReductionVar(Phi, RK_FloatMult, TheLoop, HasFunNoNaNAttr, RedDes)) {
350 DEBUG(
dbgs() <<
"Found an FMult reduction PHI." << *Phi <<
"\n");
353 if (AddReductionVar(Phi, RK_FloatAdd, TheLoop, HasFunNoNaNAttr, RedDes)) {
354 DEBUG(
dbgs() <<
"Found an FAdd reduction PHI." << *Phi <<
"\n");
357 if (AddReductionVar(Phi, RK_FloatMinMax, TheLoop, HasFunNoNaNAttr, RedDes)) {
358 DEBUG(
dbgs() <<
"Found an float MINMAX reduction PHI." << *Phi <<
"\n");
396 return Instruction::Add;
398 return Instruction::Mul;
406 return Instruction::FMul;
408 return Instruction::FAdd;
409 case RK_IntegerMinMax:
410 return Instruction::ICmp;
412 return Instruction::FCmp;
446 if (RK == MRK_FloatMin || RK == MRK_FloatMax)
447 Cmp = Builder.
CreateFCmp(P, Left, Right,
"rdx.minmax.cmp");
449 Cmp = Builder.
CreateICmp(P, Left, Right,
"rdx.minmax.cmp");
466 DEBUG(
dbgs() <<
"LV: PHI is not a poly recurrence.\n");
482 assert(PhiTy->
isPointerTy() &&
"The PHI must be a pointer");
486 if (!PointerElementType->isSized())
490 int64_t Size =
static_cast<int64_t
>(DL.
getTypeAllocSize(PointerElementType));
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
ScalarEvolution - This class is the main scalar evolution driver.
bool isInductionPHI(PHINode *, ScalarEvolution *, ConstantInt *&)
Checks if the given PHINode in a loop header is an induction variable.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes)
Returns true if Phi is a reduction in TheLoop.
0 1 0 0 True if ordered and less than
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
getStepRecurrence - This method constructs and returns the recurrence indicating how much this expres...
const Function * getParent() const
Return the enclosing method, or null if none.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
MaxMin_match< FCmpInst, LHS, RHS, ufmax_pred_ty > m_UnordFMax(const LHS &L, const RHS &R)
Match an 'unordered' floating point maximum function.
static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, InstDesc &Prev, bool HasFunNoNaNAttr)
Returns a struct describing if the instruction 'I' can be a recurrence variable of type 'Kind'...
BlockT * getHeader() const
Type * getPointerElementType() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
SelectInst - This class represents the LLVM 'select' instruction.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
A Use represents the edge between a Value definition and its users.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstDesc isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev)
Returns a struct describing if the instruction if the instruction is a Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y) or max(X, Y).
MaxMin_match< FCmpInst, LHS, RHS, ufmin_pred_ty > m_UnordFMin(const LHS &L, const RHS &R)
Match an 'unordered' floating point minimum function.
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
This instruction compares its operands according to the predicate given to the constructor.
MaxMin_match< FCmpInst, LHS, RHS, ofmin_pred_ty > m_OrdFMin(const LHS &L, const RHS &R)
Match an 'ordered' floating point minimum function.
bool hasUnsafeAlgebra() const
Determine whether the unsafe-algebra flag is set.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl< Instruction * > &Set)
Returns true if all uses of the instruction I is within the Set.
bool isFloatingPointTy() const
isFloatingPointTy - Return true if this is one of the six floating point types
This POD struct holds information about a potential recurrence operation.
static bool hasMultipleUsesOf(Instruction *I, SmallPtrSetImpl< Instruction * > &Insts)
Returns true if instuction I has multiple uses in Insts.
unsigned getNumIncomingValues() const
getNumIncomingValues - Return the number of incoming edges
static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop, bool HasFunNoNaNAttr, RecurrenceDescriptor &RedDes)
Returns true if Phi is a reduction of type Kind and adds it to the RecurrenceDescriptor.
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important base class in LLVM.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
bool contains(const LoopT *L) const
contains - Return true if the specified loop is contained within in this loop.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Value * getOperand(unsigned i) const
bool isCommutative() const
isCommutative - Return true if the instruction is commutative:
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
static unsigned getRecurrenceBinOp(RecurrenceKind Kind)
Returns the opcode of binary operation corresponding to the RecurrenceKind.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
0 0 1 0 True if ordered and greater than
MinMaxRecurrenceKind getMinMaxKind()
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is the shared class of boolean and integer constants.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
ConstantInt * getValue() const
static Constant * get(Type *Ty, double V)
get() - This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in the specified type.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static Constant * getRecurrenceIdentity(RecurrenceKind K, Type *Tp)
Returns identity corresponding to the RecurrenceKind.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="")
iterator_range< user_iterator > users()
static Value * createMinMaxOp(IRBuilder<> &Builder, MinMaxRecurrenceKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
SCEV - This class represents an analyzed expression in the program.
bool hasOneUse() const
Return true if there is exactly one user of this value.
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
StringRef getValueAsString() const
Return the attribute's value as a string.
user_iterator user_begin()
const ARM::ArchExtKind Kind
MaxMin_match< FCmpInst, LHS, RHS, ofmax_pred_ty > m_OrdFMax(const LHS &L, const RHS &R)
Match an 'ordered' floating point maximum function.
LLVM Value Representation.
const SCEV * getSCEV(Value *V)
getSCEV - Return a SCEV expression for the full generality of the specified expression.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
const BasicBlock * getParent() const
RecurrenceKind
This enum represents the kinds of recurrences that we support.
SCEVConstant - This class represents a constant integer value.