33 #define DEBUG_TYPE "indvars"
35 STATISTIC(NumElimIdentity,
"Number of IV identities eliminated");
36 STATISTIC(NumElimOperand,
"Number of IV operands folded into a use");
37 STATISTIC(NumElimRem ,
"Number of IV remainder operations eliminated");
38 STATISTIC(NumElimCmp ,
"Number of IV comparisons eliminated");
45 class SimplifyIndvar {
58 :
L(Loop), LI(LI), SE(SE), DT(DT), DeadInsts(Dead), Changed(
false) {
59 assert(LI &&
"IV simplification requires LoopInfo");
62 bool hasChanged()
const {
return Changed; }
73 bool eliminateOverflowIntrinsic(
CallInst *CI);
91 Value *IVSrc =
nullptr;
93 const SCEV *FoldedExpr =
nullptr;
97 case Instruction::UDiv:
98 case Instruction::LShr:
101 if (IVOperand != UseInst->
getOperand(OperIdx) ||
107 if (!isa<BinaryOperator>(IVOperand)
108 || !isa<ConstantInt>(IVOperand->
getOperand(1)))
113 assert(SE->isSCEVable(IVSrc->
getType()) &&
"Expect SCEVable IV operand");
116 if (UseInst->
getOpcode() == Instruction::LShr) {
125 FoldedExpr = SE->getUDivExpr(SE->getSCEV(IVSrc), SE->getSCEV(D));
128 if (!SE->isSCEVable(UseInst->
getType()))
132 if (SE->getSCEV(UseInst) != FoldedExpr)
135 DEBUG(
dbgs() <<
"INDVARS: Eliminated IV operand: " << *IVOperand
136 <<
" -> " << *UseInst <<
'\n');
139 assert(SE->getSCEV(UseInst) == FoldedExpr &&
"bad SCEV with folded oper");
144 DeadInsts.emplace_back(IVOperand);
150 void SimplifyIndvar::eliminateIVComparison(
ICmpInst *ICmp,
Value *IVOperand) {
151 unsigned IVOperIdx = 0;
166 S = SE->getSCEVAtScope(S, ICmpLoop);
167 X = SE->getSCEVAtScope(X, ICmpLoop);
170 const SCEV *InvariantLHS, *InvariantRHS;
174 if (SE->isKnownPredicate(Pred, S, X)) {
176 DeadInsts.emplace_back(ICmp);
177 DEBUG(
dbgs() <<
"INDVARS: Eliminated comparison: " << *ICmp <<
'\n');
180 DeadInsts.emplace_back(ICmp);
181 DEBUG(
dbgs() <<
"INDVARS: Eliminated comparison: " << *ICmp <<
'\n');
182 }
else if (isa<PHINode>(IVOperand) &&
183 SE->isLoopInvariantPredicate(Pred, S, X,
L, InvariantPredicate,
184 InvariantLHS, InvariantRHS)) {
190 Value *NewLHS =
nullptr, *NewRHS =
nullptr;
192 if (S == InvariantLHS || X == InvariantLHS)
194 ICmp->
getOperand(S == InvariantLHS ? IVOperIdx : (1 - IVOperIdx));
196 if (S == InvariantRHS || X == InvariantRHS)
198 ICmp->
getOperand(S == InvariantRHS ? IVOperIdx : (1 - IVOperIdx));
200 auto *PN = cast<PHINode>(IVOperand);
201 for (
unsigned i = 0, e = PN->getNumIncomingValues();
202 i != e && (!NewLHS || !NewRHS);
236 Value *Incoming = PN->getIncomingValue(
i);
239 if (
auto *
I = dyn_cast<Instruction>(Incoming))
240 assert(DT->dominates(
I, ICmp) &&
"Should be a unique loop dominating value!");
243 const SCEV *IncomingS = SE->getSCEV(Incoming);
245 if (!NewLHS && IncomingS == InvariantLHS)
247 if (!NewRHS && IncomingS == InvariantRHS)
251 if (!NewLHS || !NewRHS)
257 DEBUG(
dbgs() <<
"INDVARS: Simplified comparison: " << *ICmp <<
'\n');
284 S = SE->getSCEVAtScope(S, ICmpLoop);
285 X = SE->getSCEVAtScope(X, ICmpLoop);
288 if ((!IsSigned || SE->isKnownNonNegative(S)) &&
294 const SCEV *LessOne = SE->getMinusSCEV(S, SE->getOne(S->
getType()));
295 if (IsSigned && !SE->isKnownNonNegative(LessOne))
298 if (!SE->isKnownPredicate(IsSigned ?
312 DEBUG(
dbgs() <<
"INDVARS: Simplified rem: " << *Rem <<
'\n');
315 DeadInsts.emplace_back(Rem);
318 bool SimplifyIndvar::eliminateOverflowIntrinsic(
CallInst *CI) {
328 OperationFunctionTy Operation;
329 ExtensionFunctionTy Extension;
335 bool NoSignedOverflow;
337 switch (
F->getIntrinsicID()) {
341 case Intrinsic::sadd_with_overflow:
345 NoSignedOverflow =
true;
348 case Intrinsic::uadd_with_overflow:
352 NoSignedOverflow =
false;
355 case Intrinsic::ssub_with_overflow:
358 RawOp = Instruction::Sub;
359 NoSignedOverflow =
true;
362 case Intrinsic::usub_with_overflow:
365 RawOp = Instruction::Sub;
366 NoSignedOverflow =
false;
373 auto *NarrowTy = cast<IntegerType>(LHS->
getType());
380 (SE->*Operation)((SE->*Extension)(LHS, WideTy),
392 if (NoSignedOverflow)
399 for (
auto *U : CI->
users()) {
400 if (
auto *EVI = dyn_cast<ExtractValueInst>(U)) {
401 if (EVI->getIndices()[0] == 1)
404 assert(EVI->getIndices()[0] == 0 &&
"Only two possibilities!");
405 EVI->replaceAllUsesWith(NewResult);
411 for (
auto *EVI : ToDelete)
412 EVI->eraseFromParent();
423 bool SimplifyIndvar::eliminateIVUser(
Instruction *UseInst,
425 if (
ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) {
426 eliminateIVComparison(ICmp, IVOperand);
430 bool IsSigned = Rem->
getOpcode() == Instruction::SRem;
431 if (IsSigned || Rem->
getOpcode() == Instruction::URem) {
432 eliminateIVRemainder(Rem, IVOperand, IsSigned);
437 if (
auto *CI = dyn_cast<CallInst>(UseInst))
438 if (eliminateOverflowIntrinsic(CI))
441 if (eliminateIdentitySCEV(UseInst, IVOperand))
448 bool SimplifyIndvar::eliminateIdentitySCEV(
Instruction *UseInst,
450 if (!SE->isSCEVable(UseInst->
getType()) ||
452 (SE->getSCEV(UseInst) != SE->getSCEV(IVOperand)))
471 if (isa<PHINode>(UseInst))
474 if (!DT || !DT->dominates(IVOperand, UseInst))
477 if (!LI->replacementPreservesLCSSAForm(UseInst, IVOperand))
480 DEBUG(
dbgs() <<
"INDVARS: Eliminated identity: " << *UseInst <<
'\n');
485 DeadInsts.emplace_back(UseInst);
491 bool SimplifyIndvar::strengthenOverflowingOperation(
BinaryOperator *BO,
509 case Instruction::Sub:
513 case Instruction::Mul:
523 bool Changed =
false;
526 const SCEV *ExtendAfterOp = SE->getZeroExtendExpr(SE->getSCEV(BO), WideTy);
527 const SCEV *OpAfterExtend = (SE->*GetExprForBO)(
528 SE->getZeroExtendExpr(LHS, WideTy), SE->getZeroExtendExpr(RHS, WideTy),
530 if (ExtendAfterOp == OpAfterExtend) {
538 const SCEV *ExtendAfterOp = SE->getSignExtendExpr(SE->getSCEV(BO), WideTy);
539 const SCEV *OpAfterExtend = (SE->*GetExprForBO)(
540 SE->getSignExtendExpr(LHS, WideTy), SE->getSignExtendExpr(RHS, WideTy),
542 if (ExtendAfterOp == OpAfterExtend) {
556 SmallVectorImpl< std::pair<Instruction*,Instruction*> > &SimpleIVUsers) {
565 if (UI != Def && Simplified.
insert(UI).second)
566 SimpleIVUsers.push_back(std::make_pair(UI, Def));
604 if (!SE->isSCEVable(CurrIV->
getType()))
618 while (!SimpleIVUsers.
empty()) {
619 std::pair<Instruction*, Instruction*> UseOper =
624 if (UseInst == CurrIV)
continue;
627 for (
unsigned N = 0; IVOperand; ++
N) {
628 assert(
N <= Simplified.
size() &&
"runaway iteration");
630 Value *NewOper = foldIVUser(UseOper.first, IVOperand);
638 if (eliminateIVUser(UseOper.first, IVOperand)) {
643 if (
BinaryOperator *BO = dyn_cast<BinaryOperator>(UseOper.first)) {
644 if (isa<OverflowingBinaryOperator>(BO) &&
645 strengthenOverflowingOperation(BO, IVOperand)) {
658 pushIVUsers(UseOper.first, Simplified, SimpleIVUsers);
673 SIV.simplifyUsers(CurrIV, V);
674 return SIV.hasChanged();
681 bool Changed =
false;
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void push_back(const T &Elt)
static ConstantInt * getFalse(LLVMContext &Context)
STATISTIC(NumFunctions,"Total number of functions")
The main scalar evolution driver.
This class represents a function call, abstracting a target machine's calling convention.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
BlockT * getHeader() const
Interface for visiting interesting IV users that are recognized but not simplified by this utility...
This class represents the LLVM 'select' instruction.
This is the base class for all instructions that perform data casts.
const APInt & getValue() const
Return the constant as an APInt value reference.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT, LoopInfo *LI, SmallVectorImpl< WeakVH > &Dead, IVVisitor *V=nullptr)
simplifyUsersOfIV - Simplify instructions that use this induction variable by using ScalarEvolution t...
LLVM_NODISCARD bool empty() const
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This node represents a polynomial recurrence on the trip count of the specified loop.
Function Alias Analysis false
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
The instances of the Type class are immutable: once they are created, they are never changed...
Type * getType() const
Return the LLVM type of this SCEV expression.
bool hasNoSignedWrap() const
Determine whether the no signed wrap flag is set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
This instruction compares its operands according to the predicate given to the constructor.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return LHS-RHS. Minus is represented in SCEV as A+B*-1.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Value * getOperand(unsigned i) const
Predicate getPredicate() const
Return the predicate for this instruction.
LLVMContext & getContext() const
All values hold a context through their type.
BinaryOps getOpcode() const
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
void setHasNoSignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag...
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is the shared class of boolean and integer constants.
bool hasNoUnsignedWrap() const
Determine whether the no unsigned wrap flag is set.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
LLVM_NODISCARD T pop_back_val()
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
static ConstantInt * getTrue(LLVMContext &Context)
void setPredicate(Predicate P)
Set the predicate for this instruction to the specified value.
void setOperand(unsigned i, Value *Val)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static bool isSimpleIVUser(Instruction *I, const Loop *L, ScalarEvolution *SE)
Return true if this instruction generates a simple SCEV expression in terms of that IV...
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty)
bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, DominatorTree *DT, LoopInfo *LI, SmallVectorImpl< WeakVH > &Dead)
SimplifyLoopIVs - Simplify users of induction variables within this loop.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
iterator_range< user_iterator > users()
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical add expression, or something simpler if possible.
This class represents an analyzed expression in the program.
virtual void visitCast(CastInst *Cast)=0
Represents a single loop in the control flow graph.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
const Loop * getLoop() const
void setHasNoUnsignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void pushIVUsers(Instruction *Def, SmallPtrSet< Instruction *, 16 > &Simplified, SmallVectorImpl< std::pair< Instruction *, Instruction * > > &SimpleIVUsers)
Add all uses of Def to the current IV's worklist.
LLVM Value Representation.
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty)
NoWrapFlags
NoWrapFlags are bitfield indices into SubclassData.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical multiply expression, or something simpler if possible.
const BasicBlock * getParent() const