54 #define DEBUG_TYPE "indvars"
56 STATISTIC(NumWidened ,
"Number of indvars widened");
57 STATISTIC(NumReplaced ,
"Number of exit values replaced");
58 STATISTIC(NumLFTR ,
"Number of loop exit tests replaced");
59 STATISTIC(NumElimExt ,
"Number of IV sign/zero extends eliminated");
60 STATISTIC(NumElimIV ,
"Number of congruent IVs eliminated");
67 cl::desc(
"Verify the ScalarEvolution result after running indvars"));
70 cl::desc(
"Reduce live induction variables."));
76 cl::desc(
"Choose the strategy to replace exit value in IndVarSimplify"),
79 "only replace exit value when the cost is cheap"),
81 "always replace exit value whenever possible"),
89 class IndVarSimplify :
public LoopPass {
102 :
LoopPass(
ID), LI(nullptr), SE(nullptr), DT(nullptr), Changed(
false) {
121 void releaseMemory()
override {
125 bool isValidRewrite(
Value *FromVal,
Value *ToVal);
128 void RewriteNonIntegerIVs(
Loop *L);
135 Value *LinearFunctionTestReplace(
Loop *L,
const SCEV *BackedgeTakenCount,
138 void SinkUnusedInvariants(
Loop *L);
142 bool &IsHighCostExpansion);
148 "Induction Variable Simplification",
false,
false)
158 return new IndVarSimplify();
165 bool IndVarSimplify::isValidRewrite(
Value *FromVal,
Value *ToVal) {
176 Value *FromPtr = FromVal;
177 Value *ToPtr = ToVal;
179 FromPtr =
GEP->getPointerOperand();
182 ToPtr =
GEP->getPointerOperand();
184 if (FromPtr != FromVal || ToPtr != ToVal) {
186 if (FromPtr == ToPtr)
200 const SCEV *FromBase = SE->getPointerBase(SE->getSCEV(FromPtr));
201 const SCEV *ToBase = SE->getPointerBase(SE->getSCEV(ToPtr));
202 if (FromBase == ToBase)
205 DEBUG(
dbgs() <<
"INDVARS: GEP rewrite bail out "
206 << *FromBase <<
" != " << *ToBase <<
"\n");
236 assert(InsertPt &&
"Missing phi operand");
237 assert((!isa<Instruction>(Def) ||
238 DT->
dominates(cast<Instruction>(Def), InsertPt)) &&
239 "def does not dominate all uses");
249 bool isExact =
false;
268 void IndVarSimplify::HandleFloatingPointIV(
Loop *L,
PHINode *PN) {
270 unsigned BackEdge = IncomingEdge^1;
284 if (Incr ==
nullptr || Incr->
getOpcode() != Instruction::FAdd)
return;
290 if (IncValueVal ==
nullptr || Incr->
getOperand(0) != PN ||
298 if (IncrUse == Incr->
user_end())
return;
300 if (IncrUse != Incr->
user_end())
return;
317 assert(TheBr->
isConditional() &&
"Can't use fcmp if not conditional");
328 if (ExitValueVal ==
nullptr ||
368 if (InitValue >= ExitValue)
371 uint32_t Range = uint32_t(ExitValue-InitValue);
375 if (++Range == 0)
return;
378 unsigned Leftover = Range % uint32_t(IncValue);
389 if (Leftover != 0 && int32_t(ExitValue+IncValue) < ExitValue)
395 if (InitValue <= ExitValue)
398 uint32_t Range = uint32_t(InitValue-ExitValue);
402 if (++Range == 0)
return;
405 unsigned Leftover = Range % uint32_t(-IncValue);
416 if (Leftover != 0 && int32_t(ExitValue+IncValue) > ExitValue)
466 void IndVarSimplify::RewriteNonIntegerIVs(
Loop *L) {
478 for (
unsigned i = 0, e = PHIs.
size(); i != e; ++i)
479 if (
PHINode *PN = dyn_cast_or_null<PHINode>(&*PHIs[i]))
480 HandleFloatingPointIV(L, PN);
500 : PN(P), Ith(I), Val(V), HighCost(H), SafePhi(S) {}
507 bool &IsHighCostExpansion) {
508 using namespace llvm::PatternMatch;
511 IsHighCostExpansion =
false;
528 if (!
match(BB->getTerminator(),
533 if (SE->getSCEV(LHS) == S && DT->dominates(LHS, InsertPt)) {
534 IsHighCostExpansion =
false;
538 if (SE->getSCEV(RHS) == S && DT->dominates(RHS, InsertPt)) {
539 IsHighCostExpansion =
false;
546 IsHighCostExpansion =
true;
565 void IndVarSimplify::RewriteLoopExitValues(
Loop *L,
SCEVExpander &Rewriter) {
576 for (
unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
597 bool LCSSASafePhiForRAUW =
603 while ((PN = dyn_cast<PHINode>(BBI++))) {
619 for (
unsigned i = 0; i != NumPreds; ++i) {
623 if (!isa<Instruction>(InVal))
639 if (!SE->isLoopInvariant(ExitValue, L) ||
649 unsigned NumHardInternalUses = 0;
650 unsigned NumSoftExternalUses = 0;
651 unsigned NumUses = 0;
653 IB !=
IE && NumUses <= 6; ++IB) {
659 NumHardInternalUses++;
667 PB != PE && NumUses <= 6; ++PB, ++NumUses) {
668 unsigned PhiOpc = cast<Instruction>(*PB)->getOpcode();
670 NumSoftExternalUses++;
675 NumSoftExternalUses++;
678 if (NumUses <= 6 && NumHardInternalUses && !NumSoftExternalUses)
682 bool HighCost =
false;
683 Value *ExitVal = ExpandSCEVIfNeeded(Rewriter, ExitValue, L, Inst,
686 DEBUG(
dbgs() <<
"INDVARS: RLEV: AfterLoopVal = " << *ExitVal <<
'\n'
687 <<
" LoopVal = " << *Inst <<
"\n");
689 if (!isValidRewrite(Inst, ExitVal)) {
690 DeadInsts.push_back(ExitVal);
696 RewritePhi(PN, i, ExitVal, HighCost, LCSSASafePhiForRAUW));
701 bool LoopCanBeDel = CanLoopBeDeleted(L, RewritePhiSet);
704 for (
const RewritePhi &Phi : RewritePhiSet) {
706 Value *ExitVal = Phi.Val;
711 DeadInsts.push_back(ExitVal);
723 DeadInsts.push_back(Inst);
741 bool IndVarSimplify::CanLoopBeDeleted(
757 if (ExitBlocks.
size() > 1 || ExitingBlocks.
size() > 1)
762 while (
PHINode *
P = dyn_cast<PHINode>(BI)) {
763 Value *Incoming =
P->getIncomingValueForBlock(ExitingBlocks[0]);
769 for (
const RewritePhi &Phi : RewritePhiSet) {
770 unsigned i = Phi.Ith;
771 if (Phi.PN ==
P && (Phi.PN)->getIncomingValue(i) == Incoming) {
778 if (!found && (I = dyn_cast<Instruction>(Incoming)))
789 if (BI->mayHaveSideEffects())
807 Type *WidestNativeType;
810 WideIVInfo() : NarrowIV(nullptr), WidestNativeType(nullptr),
820 bool IsSigned = Cast->
getOpcode() == Instruction::SExt;
821 if (!IsSigned && Cast->
getOpcode() != Instruction::ZExt)
842 if (!WI.WidestNativeType) {
844 WI.IsSigned = IsSigned;
849 if (WI.IsSigned != IsSigned)
861 struct NarrowIVDefUse {
866 NarrowIVDefUse(): NarrowDef(nullptr), NarrowUse(nullptr), WideDef(nullptr) {}
869 NarrowDef(ND), NarrowUse(NU), WideDef(WD) {}
892 const SCEV *WideIncExpr;
899 WidenIV(
const WideIVInfo &WI,
LoopInfo *LInfo,
902 OrigPhi(WI.NarrowIV),
903 WideType(WI.WidestNativeType),
904 IsSigned(WI.IsSigned),
911 WideIncExpr(nullptr),
913 assert(L->
getHeader() == OrigPhi->getParent() &&
"Phi must be an IV");
919 Value *getExtend(
Value *NarrowOper,
Type *WideType,
bool IsSigned,
926 const SCEVAddRecExpr* GetExtendedOperandRecurrence(NarrowIVDefUse DU);
928 const SCEV *GetSCEVByOpCode(
const SCEV *LHS,
const SCEV *RHS,
929 unsigned OpCode)
const;
933 bool WidenLoopCompare(NarrowIVDefUse DU);
950 Value *WidenIV::getExtend(
Value *NarrowOper,
Type *WideType,
bool IsSigned,
960 return IsSigned ? Builder.CreateSExt(NarrowOper, WideType) :
961 Builder.CreateZExt(NarrowOper, WideType);
967 Instruction *WidenIV::CloneIVUser(NarrowIVDefUse DU) {
968 unsigned Opcode = DU.NarrowUse->getOpcode();
972 case Instruction::Add:
973 case Instruction::Mul:
974 case Instruction::UDiv:
975 case Instruction::Sub:
979 case Instruction::Shl:
980 case Instruction::LShr:
981 case Instruction::AShr:
982 DEBUG(
dbgs() <<
"Cloning IVUser: " << *DU.NarrowUse <<
"\n");
989 Value *LHS = (DU.NarrowUse->getOperand(0) == DU.NarrowDef) ? DU.WideDef :
990 getExtend(DU.NarrowUse->getOperand(0), WideType, IsSigned, DU.NarrowUse);
991 Value *RHS = (DU.NarrowUse->getOperand(1) == DU.NarrowDef) ? DU.WideDef :
992 getExtend(DU.NarrowUse->getOperand(1), WideType, IsSigned, DU.NarrowUse);
999 Builder.Insert(WideBO);
1001 dyn_cast<OverflowingBinaryOperator>(NarrowBO)) {
1002 if (OBO->hasNoUnsignedWrap()) WideBO->setHasNoUnsignedWrap();
1003 if (OBO->hasNoSignedWrap()) WideBO->setHasNoSignedWrap();
1009 const SCEV *WidenIV::GetSCEVByOpCode(
const SCEV *LHS,
const SCEV *RHS,
1010 unsigned OpCode)
const {
1011 if (OpCode == Instruction::Add)
1012 return SE->getAddExpr(LHS, RHS);
1013 if (OpCode == Instruction::Sub)
1014 return SE->getMinusSCEV(LHS, RHS);
1015 if (OpCode == Instruction::Mul)
1016 return SE->getMulExpr(LHS, RHS);
1025 const SCEVAddRecExpr* WidenIV::GetExtendedOperandRecurrence(NarrowIVDefUse DU) {
1028 const unsigned OpCode = DU.NarrowUse->getOpcode();
1030 if (OpCode != Instruction::Add && OpCode != Instruction::Sub &&
1031 OpCode != Instruction::Mul)
1036 const unsigned ExtendOperIdx =
1037 DU.NarrowUse->getOperand(0) == DU.NarrowDef ? 1 : 0;
1038 assert(DU.NarrowUse->getOperand(1-ExtendOperIdx) == DU.NarrowDef &&
"bad DU");
1040 const SCEV *ExtendOperExpr =
nullptr;
1042 cast<OverflowingBinaryOperator>(DU.NarrowUse);
1044 ExtendOperExpr = SE->getSignExtendExpr(
1045 SE->getSCEV(DU.NarrowUse->getOperand(ExtendOperIdx)), WideType);
1047 ExtendOperExpr = SE->getZeroExtendExpr(
1048 SE->getSCEV(DU.NarrowUse->getOperand(ExtendOperIdx)), WideType);
1057 const SCEV *lhs = SE->getSCEV(DU.WideDef);
1058 const SCEV *rhs = ExtendOperExpr;
1062 if (ExtendOperIdx == 0)
1067 if (!AddRec || AddRec->
getLoop() != L)
1078 if (!SE->isSCEVable(NarrowUse->
getType()))
1081 const SCEV *NarrowExpr = SE->getSCEV(NarrowUse);
1082 if (SE->getTypeSizeInBits(NarrowExpr->
getType())
1083 >= SE->getTypeSizeInBits(WideType)) {
1089 const SCEV *WideExpr = IsSigned ?
1090 SE->getSignExtendExpr(NarrowExpr, WideType) :
1091 SE->getZeroExtendExpr(NarrowExpr, WideType);
1093 if (!AddRec || AddRec->
getLoop() != L)
1101 DEBUG(
dbgs() <<
"INDVARS: Truncate IV " << *DU.WideDef
1102 <<
" for user " << *DU.NarrowUse <<
"\n");
1105 DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, Trunc);
1111 bool WidenIV::WidenLoopCompare(NarrowIVDefUse DU) {
1121 unsigned CastWidth = SE->getTypeSizeInBits(Op->
getType());
1122 unsigned IVWidth = SE->getTypeSizeInBits(WideType);
1123 assert (CastWidth <= IVWidth &&
"Unexpected width while widening compare.");
1127 DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, DU.WideDef);
1130 if (CastWidth < IVWidth) {
1131 Value *ExtOp = getExtend(Op, WideType, IsSigned, Cmp);
1132 DU.NarrowUse->replaceUsesOfWith(Op, ExtOp);
1142 if (
PHINode *UsePhi = dyn_cast<PHINode>(DU.NarrowUse)) {
1143 if (LI->getLoopFor(UsePhi->getParent()) != L) {
1147 if (UsePhi->getNumOperands() != 1)
1151 PHINode::Create(DU.WideDef->getType(), 1, UsePhi->getName() +
".wide",
1153 WidePhi->
addIncoming(DU.WideDef, UsePhi->getIncomingBlock(0));
1155 Value *Trunc = Builder.CreateTrunc(WidePhi, DU.NarrowDef->getType());
1157 DeadInsts.emplace_back(UsePhi);
1158 DEBUG(
dbgs() <<
"INDVARS: Widen lcssa phi " << *UsePhi
1159 <<
" to " << *WidePhi <<
"\n");
1165 if (IsSigned ? isa<SExtInst>(DU.NarrowUse) : isa<ZExtInst>(DU.NarrowUse)) {
1166 Value *NewDef = DU.WideDef;
1167 if (DU.NarrowUse->getType() != WideType) {
1168 unsigned CastWidth = SE->getTypeSizeInBits(DU.NarrowUse->getType());
1169 unsigned IVWidth = SE->getTypeSizeInBits(WideType);
1170 if (CastWidth < IVWidth) {
1173 NewDef = Builder.CreateTrunc(DU.WideDef, DU.NarrowUse->getType());
1179 DEBUG(
dbgs() <<
"INDVARS: New IV " << *WidePhi
1180 <<
" not wide enough to subsume " << *DU.NarrowUse <<
"\n");
1181 DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, DU.WideDef);
1182 NewDef = DU.NarrowUse;
1185 if (NewDef != DU.NarrowUse) {
1186 DEBUG(
dbgs() <<
"INDVARS: eliminating " << *DU.NarrowUse
1187 <<
" replaced by " << *DU.WideDef <<
"\n");
1189 DU.NarrowUse->replaceAllUsesWith(NewDef);
1190 DeadInsts.emplace_back(DU.NarrowUse);
1203 const SCEVAddRecExpr *WideAddRec = GetWideRecurrence(DU.NarrowUse);
1205 WideAddRec = GetExtendedOperandRecurrence(DU);
1210 if (WidenLoopCompare(DU))
1221 assert(DU.NarrowUse != DU.NarrowUse->getParent()->getTerminator() &&
1222 "SCEV is not expected to evaluate a block terminator");
1227 if (WideAddRec == WideIncExpr
1228 && Rewriter.
hoistIVInc(WideInc, DU.NarrowUse))
1231 WideUse = CloneIVUser(DU);
1240 if (WideAddRec != SE->getSCEV(WideUse)) {
1241 DEBUG(
dbgs() <<
"Wide use expression mismatch: " << *WideUse
1242 <<
": " << *SE->getSCEV(WideUse) <<
" != " << *WideAddRec <<
"\n");
1243 DeadInsts.emplace_back(WideUse);
1258 if (!Widened.insert(NarrowUser).second)
1261 NarrowIVUsers.push_back(NarrowIVDefUse(NarrowDef, NarrowUser, WideDef));
1282 const SCEV *WideIVExpr = IsSigned ?
1283 SE->getSignExtendExpr(AddRec, WideType) :
1284 SE->getZeroExtendExpr(AddRec, WideType);
1286 assert(SE->getEffectiveSCEVType(WideIVExpr->
getType()) == WideType &&
1287 "Expect the new IV expression to preserve its type");
1291 if (!AddRec || AddRec->
getLoop() != L)
1299 &&
"Loop header phi recurrence inputs do not dominate the loop");
1306 WidePhi = cast<PHINode>(Rewriter.
expandCodeFor(AddRec, WideType, InsertPt));
1315 WideIncExpr = SE->getSCEV(WideInc);
1318 DEBUG(
dbgs() <<
"Wide IV: " << *WidePhi <<
"\n");
1322 assert(Widened.empty() && NarrowIVUsers.empty() &&
"expect initial state" );
1324 Widened.insert(OrigPhi);
1325 pushNarrowIVUsers(OrigPhi, WidePhi);
1327 while (!NarrowIVUsers.empty()) {
1328 NarrowIVDefUse DU = NarrowIVUsers.pop_back_val();
1336 pushNarrowIVUsers(DU.NarrowUse, WideUse);
1339 if (DU.NarrowDef->use_empty())
1340 DeadInsts.emplace_back(DU.NarrowDef);
1355 class IndVarSimplifyVisitor :
public IVVisitor {
1366 : SE(SCEV), TTI(TTI), IVPhi(IV) {
1368 WI.NarrowIV = IVPhi;
1370 setSplitOverflowIntrinsics();
1384 void IndVarSimplify::SimplifyAndExtend(
Loop *L,
1397 while (!LoopPhis.
empty()) {
1408 IndVarSimplifyVisitor Visitor(CurrIV, SE, TTI, DT);
1412 if (Visitor.WI.WidestNativeType) {
1415 }
while(!LoopPhis.
empty());
1418 WidenIV Widener(WideIVs.
back(), LI, SE, DT, DeadInsts);
1419 if (
PHINode *WidePhi = Widener.CreateWideIV(Rewriter)) {
1447 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount) ||
1448 BackedgeTakenCount->
isZero())
1472 case Instruction::Add:
1473 case Instruction::Sub:
1475 case Instruction::GetElementPtr:
1489 if (IncI->
getOpcode() == Instruction::GetElementPtr)
1511 assert(BI &&
"expected exit branch");
1560 if (isa<Constant>(V))
1561 return !isa<UndefValue>(V);
1578 if (!Visited.
insert(*OI).second)
1604 if (U != Cond && U != IncV)
return false;
1607 if (U != Cond && U != Phi)
return false;
1629 cast<BranchInst>(L->
getExitingBlock()->getTerminator())->getCondition();
1633 const SCEV *BestInit =
nullptr;
1635 assert(LatchBlock &&
"needsLFTR should guarantee a loop latch");
1654 if (PhiWidth < BCWidth ||
1655 !L->
getHeader()->getModule()->getDataLayout().isLegalInteger(PhiWidth))
1659 if (!Step || !Step->
isOne())
1682 if (BestPhi && !
AlmostDeadIV(BestPhi, LatchBlock, Cond)) {
1696 else if (PhiWidth <= SE->getTypeSizeInBits(BestPhi->
getType()))
1710 assert(AR && AR->
getLoop() == L && AR->
isAffine() &&
"bad loop counter");
1731 "Computed iteration count is not loop invariant!");
1740 cast<PointerType>(GEPBase->
getType())->getElementType())->isOne()
1741 &&
"unit stride pointer IV must be i8*");
1744 return Builder.
CreateGEP(
nullptr, GEPBase, GEPOffset,
"lftr.limit");
1758 const SCEV *IVLimit =
nullptr;
1778 "Computed iteration count is not loop invariant!");
1793 Value *IndVarSimplify::
1794 LinearFunctionTestReplace(
Loop *L,
1795 const SCEV *BackedgeTakenCount,
1801 Value *CmpIndVar = IndVar;
1802 const SCEV *IVCount = BackedgeTakenCount;
1811 IVCount = SE->getAddExpr(BackedgeTakenCount,
1812 SE->getConstant(BackedgeTakenCount->
getType(), 1));
1821 &&
"genLoopLimit missed a cast");
1826 if (L->
contains(BI->getSuccessor(0)))
1831 DEBUG(
dbgs() <<
"INDVARS: Rewriting loop exit condition to:\n"
1832 <<
" LHS:" << *CmpIndVar <<
'\n'
1835 <<
" RHS:\t" << *ExitCnt <<
"\n"
1836 <<
" IVCount:\t" << *IVCount <<
"\n");
1842 unsigned CmpIndVarSize = SE->getTypeSizeInBits(CmpIndVar->
getType());
1843 unsigned ExitCntSize = SE->getTypeSizeInBits(ExitCnt->
getType());
1844 if (CmpIndVarSize > ExitCntSize) {
1845 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(SE->getSCEV(IndVar));
1849 if (isa<SCEVConstant>(ARStart) && isa<SCEVConstant>(IVCount)) {
1850 const APInt &Start = cast<SCEVConstant>(ARStart)->getValue()->getValue();
1851 APInt Count = cast<SCEVConstant>(IVCount)->getValue()->getValue();
1854 if (IVCount != BackedgeTakenCount && Count == 0) {
1859 Count = Count.
zext(CmpIndVarSize);
1861 if (cast<SCEVConstant>(ARStep)->getValue()->isNegative())
1862 NewLimit = Start - Count;
1864 NewLimit = Start + Count;
1867 DEBUG(
dbgs() <<
" Widen RHS:\t" << *ExitCnt <<
"\n");
1869 CmpIndVar = Builder.CreateTrunc(CmpIndVar, ExitCnt->
getType(),
1873 Value *Cond = Builder.CreateICmp(P, CmpIndVar, ExitCnt,
"exitcond");
1874 Value *OrigCond = BI->getCondition();
1880 BI->setCondition(Cond);
1881 DeadInsts.push_back(OrigCond);
1895 void IndVarSimplify::SinkUnusedInvariants(
Loop *L) {
1897 if (!ExitBlock)
return;
1900 if (!Preheader)
return;
1904 while (I != Preheader->
begin()) {
1907 if (isa<PHINode>(I))
1916 if (I->mayHaveSideEffects() || I->mayReadFromMemory())
1920 if (isa<DbgInfoIntrinsic>(I))
1924 if (isa<LandingPadInst>(I))
1931 if (isa<AllocaInst>(I))
1936 bool UsedInLoop =
false;
1937 for (Use &U : I->uses()) {
1940 if (
PHINode *P = dyn_cast<PHINode>(User)) {
1943 UseBB = P->getIncomingBlock(i);
1945 if (UseBB == Preheader || L->
contains(UseBB)) {
1959 if (I != Preheader->
begin()) {
1963 }
while (isa<DbgInfoIntrinsic>(I) && I != Preheader->
begin());
1965 if (isa<DbgInfoIntrinsic>(I) && I == Preheader->
begin())
1982 if (skipOptnoneFunction(L))
1996 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1997 SE = &getAnalysis<ScalarEvolution>();
1998 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1999 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
2000 TLI = TLIP ? &TLIP->getTLI() :
nullptr;
2001 auto *TTIP = getAnalysisIfAvailable<TargetTransformInfoWrapperPass>();
2002 TTI = TTIP ? &TTIP->getTTI(*L->
getHeader()->getParent()) :
nullptr;
2010 RewriteNonIntegerIVs(L);
2012 const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(L);
2027 SimplifyAndExtend(L, Rewriter, LPM);
2036 !isa<SCEVCouldNotCompute>(BackedgeTakenCount))
2037 RewriteLoopExitValues(L, Rewriter);
2057 (
void)LinearFunctionTestReplace(L, BackedgeTakenCount, IndVar,
2068 while (!DeadInsts.empty())
2070 dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val()))
2077 SinkUnusedInvariants(L);
2083 "Indvars did not leave the loop in lcssa form!");
2088 if (
VerifyIndvars && !isa<SCEVCouldNotCompute>(BackedgeTakenCount)) {
2090 const SCEV *NewBECount = SE->getBackedgeTakenCount(L);
2091 if (SE->getTypeSizeInBits(BackedgeTakenCount->
getType()) <
2092 SE->getTypeSizeInBits(NewBECount->
getType()))
2093 NewBECount = SE->getTruncateOrNoop(NewBECount,
2094 BackedgeTakenCount->
getType());
2096 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount,
2098 assert(BackedgeTakenCount == NewBECount &&
"indvars must preserve SCEV");
bool isInt< 32 >(int64_t x)
Pass interface - Implemented by all 'passes'.
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
ValuesClass< DataType > LLVM_END_WITH_NULL values(const char *Arg, DataType Val, const char *Desc,...)
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
bool hoistIVInc(Instruction *IncV, Instruction *InsertPos)
Utility for hoisting an IV increment.
Induction Variable Simplification
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static bool isLoopInvariant(Value *V, const Loop *L, const DominatorTree *DT)
isLoopInvariant - Perform a quick domtree based check for loop invariance assuming that V is used wit...
bool isOne() const
isOne - Return true if the expression is a constant one.
STATISTIC(NumFunctions,"Total number of functions")
void getLoopLatches(SmallVectorImpl< BlockT * > &LoopLatches) const
getLoopLatches - Return all loop latch blocks of this loop.
bool isZero() const
isZero - Return true if the expression is a constant zero.
unsigned getNumOperands() const
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property...
ScalarEvolution - This class is the main scalar evolution driver.
static Instruction * getInsertPointForUses(Instruction *User, Value *Def, DominatorTree *DT)
Determine the insertion point for this user.
bool isSigned() const
Determine if this instruction is using a signed comparison.
0 1 0 0 True if ordered and less than
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
getStepRecurrence - This method constructs and returns the recurrence indicating how much this expres...
static void truncateIVUse(NarrowIVDefUse DU, DominatorTree *DT)
This IV user cannot be widen.
1 1 1 0 True if unordered or not equal
void setDebugType(const char *s)
bool isLoopInvariant(const SCEV *S, const Loop *L)
isLoopInvariant - Return true if the value of the given SCEV is unchanging in the specified loop...
LoopT * getParentLoop() const
bool hasLoopInvariantOperands(const Instruction *I) const
hasLoopInvariantOperands - Return true if all the operands of the specified instruction are loop inva...
static IntegerType * getInt64Ty(LLVMContext &C)
BlockT * getExitBlock() const
getExitBlock - If getExitBlocks would return exactly one block, return that block.
static bool hasConcreteDef(Value *V)
Return true if the given value is concrete.
BlockT * getHeader() const
static cl::opt< bool > ReduceLiveIVs("liv-reduce", cl::Hidden, cl::desc("Reduce live induction variables."))
const SCEV * getStart() const
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
getLoopLatch - If there is a single latch block for this loop, return it.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
iterator begin()
Instruction iterator methods.
1 0 0 1 True if unordered or equal
bool match(Val *V, const Pattern &P)
static Value * genLoopLimit(PHINode *IndVar, const SCEV *IVCount, Loop *L, SCEVExpander &Rewriter, ScalarEvolution *SE)
genLoopLimit - Help LinearFunctionTestReplace by generating a value that holds the RHS of the new loo...
Interface for visiting interesting IV users that are recognized but not simplified by this utility...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
This is the base class for all instructions that perform data casts.
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
opStatus convertToInteger(integerPart *, unsigned int, bool, roundingMode, bool *) const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
A Use represents the edge between a Value definition and its users.
static bool ConvertToSInt(const APFloat &APF, int64_t &IntVal)
ConvertToSInt - Convert APF to an integer, if possible.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
0 1 0 1 True if ordered and less than or equal
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
getExitingBlocks - Return all blocks inside the loop that have successors outside of the loop...
uint64_t getTypeSizeInBits(Type *Ty) const
getTypeSizeInBits - Return the size in bits of the specified type, for which isSCEVable must return t...
bool isLoopSimplifyForm() const
isLoopSimplifyForm - Return true if the Loop is in the form that the LoopSimplify form transforms loo...
user_iterator_impl< User > user_iterator
bool mayReadFromMemory() const
mayReadFromMemory - Return true if this instruction may read memory.
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Value handle that is nullable, but tries to track the Value.
This instruction compares its operands according to the predicate given to the constructor.
static cl::opt< bool > VerifyIndvars("verify-indvars", cl::Hidden, cl::desc("Verify the ScalarEvolution result after running indvars"))
BasicBlock * getSuccessor(unsigned i) const
bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
DeleteDeadPHIs - Examine each PHI in the given block and delete it if it is dead. ...
const SCEV * getSizeOfExpr(Type *IntTy, Type *AllocTy)
getSizeOfExpr - Return an expression for sizeof AllocTy that is type IntTy
AnalysisUsage & addPreservedID(const void *ID)
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Type * getEffectiveSCEVType(Type *Ty) const
getEffectiveSCEVType - Return a type with the same bitwidth as the given type and which represents ho...
void takeName(Value *V)
Transfer the name from V to this value.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, Instruction *InsertBefore, Value *FlagsOp)
unsigned getNumIncomingValues() const
getNumIncomingValues - Return the number of incoming edges
void clearInsertPoint()
Clear the current insertion point.
void clear()
Erase the contents of the InsertedExpressions map so that users trying to expand the same expression ...
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
initializer< Ty > init(const Ty &Val)
static cl::opt< ReplaceExitVal > ReplaceExitValue("replexitval", cl::Hidden, cl::init(OnlyCheapRepl), cl::desc("Choose the strategy to replace exit value in IndVarSimplify"), cl::values(clEnumValN(NeverRepl,"never","never replace exit value"), clEnumValN(OnlyCheapRepl,"cheap","only replace exit value when the cost is cheap"), clEnumValN(AlwaysRepl,"always","always replace exit value whenever possible"), clEnumValEnd))
bool isAffine() const
isAffine - Return true if this represents an expression A + B*x where A and B are loop invariant valu...
bool isSCEVable(Type *Ty) const
isSCEVable - Test if values of the given type are analyzable within the SCEV framework.
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
static bool canExpandBackedgeTakenCount(Loop *L, ScalarEvolution *SE, SCEVExpander &Rewriter)
canExpandBackedgeTakenCount - Return true if this loop's backedge taken count expression can be safel...
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
Type * getType() const
getType - Return the LLVM type of this SCEV expression.
BranchInst - Conditional or Unconditional Branch instruction.
bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, LPPassManager *LPM, SmallVectorImpl< WeakVH > &Dead, IVVisitor *V=nullptr)
simplifyUsersOfIV - Simplify instructions that use this induction variable by using ScalarEvolution t...
bool isHighCostExpansion(const SCEV *Expr, Loop *L)
Return true for expressions that may incur non-trivial cost to evaluate at runtime.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
ConstantFP - Floating Point Values [float, double].
static void visitIVCast(CastInst *Cast, WideIVInfo &WI, ScalarEvolution *SE, const TargetTransformInfo *TTI)
visitCast - Update information about the induction variable that is extended by this sign or zero ext...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
brc_match< Cond_t > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
Represent the analysis usage information of a pass.
BasicBlock * getIncomingBlock(unsigned i) const
getIncomingBlock - Return incoming basic block number i.
bool contains(const LoopT *L) const
contains - Return true if the specified loop is contained within in this loop.
This instruction compares its operands according to the predicate given to the constructor.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Utility class for integer arithmetic operators which may exhibit overflow - Add, Sub, and Mul.
unsigned getBitWidth() const
Return the number of bits in the APInt.
Value * expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I)
Insert code to directly compute the specified SCEV expression into the program.
BlockT * getExitingBlock() const
getExitingBlock - If getExitingBlocks would return exactly one block, return that block...
Value * getOperand(unsigned i) const
NodeT * findNearestCommonDominator(NodeT *A, NodeT *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B...
Class to represent integer types.
Predicate getPredicate() const
Return the predicate for this instruction.
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a trivially dead instruction...
void getUniqueExitBlocks(SmallVectorImpl< BasicBlock * > &ExitBlocks) const
getUniqueExitBlocks - Return all unique successor blocks of this loop.
LLVMContext & getContext() const
All values hold a context through their type.
static unsigned getIncomingValueNumForOperand(unsigned i)
static ICmpInst * getLoopTest(Loop *L)
Return the compare guarding the loop latch, or NULL for unrecognized tests.
1 1 0 1 True if unordered, less than, or equal
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
bool isConditional() const
0 0 1 0 True if ordered and greater than
INITIALIZE_PASS_BEGIN(IndVarSimplify,"indvars","Induction Variable Simplification", false, false) INITIALIZE_PASS_END(IndVarSimplify
unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT, SmallVectorImpl< WeakVH > &DeadInsts, const TargetTransformInfo *TTI=nullptr)
replace congruent phis with their most canonical representative.
BinaryOps getOpcode() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Value * getIncomingValue(unsigned i) const
getIncomingValue - Return incoming value number x
const SCEV * getTruncateExpr(const SCEV *Op, Type *Ty)
AnalysisUsage & addRequiredID(const void *ID)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
bool isLCSSAForm(DominatorTree &DT) const
isLCSSAForm - Return true if the Loop is in LCSSA form
1 1 0 0 True if unordered or less than
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
Instruction * user_back()
user_back - Specialize the methods defined in Value, as we know that an instruction can only be used ...
Provides information about what library functions are available for the current target.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static PHINode * getLoopPhiForCounter(Value *IncV, Loop *L, DominatorTree *DT)
getLoopPhiForCounter - Return the loop header phi IFF IncV adds a loop invariant value to the phi...
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
void setPreservesCFG()
This function should be called by the pass, iff they do not:
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Class for arbitrary precision integers.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
iterator_range< user_iterator > users()
void initializeIndVarSimplifyPass(PassRegistry &)
This class uses information about analyze scalars to rewrite expressions in canonical form...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getAddExpr - Get a canonical add expression, or something simpler if possible.
std::vector< BlockT * >::const_iterator block_iterator
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
block_iterator block_end() const
Virtual Register Rewriter
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Value * getCondition() const
SCEV - This class represents an analyzed expression in the program.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isLegalInteger(unsigned Width) const
Returns true if the specified type is known to be a native integer type supported by the CPU...
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool hasOneUse() const
Return true if there is exactly one user of this value.
0 1 1 0 True if ordered and operands are unequal
static bool needsLFTR(Loop *L, DominatorTree *DT)
needsLFTR - LinearFunctionTestReplace policy.
const Loop * getLoop() const
1 0 1 0 True if unordered or greater than
const APFloat & getValueAPF() const
const SCEV * getBackedgeTakenCount(const Loop *L)
getBackedgeTakenCount - If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCouldNotCompute object.
This class represents a cast from signed integer to floating point.
user_iterator user_begin()
unsigned getSCEVType() const
0 0 0 1 True if ordered and equal
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
isInstructionTriviallyDead - Return true if the result produced by the instruction is not used...
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
LLVM Value Representation.
1 0 1 1 True if unordered, greater than, or equal
const SCEV * getSCEV(Value *V)
getSCEV - Return a SCEV expression for the full generality of the specified expression.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
static bool AlmostDeadIV(PHINode *Phi, BasicBlock *LatchBlock, Value *Cond)
AlmostDeadIV - Return true if this IV has any uses other than the (soon to be rewritten) loop exit te...
static const Function * getParent(const Value *V)
void moveBefore(Instruction *MovePos)
moveBefore - Unlink this instruction from its current basic block and insert it into the basic block ...
void disableCanonicalMode()
Disable the behavior of expanding expressions in canonical form rather than in a more literal form...
block_iterator block_begin() const
The legacy pass manager's analysis pass to compute loop information.
APInt LLVM_ATTRIBUTE_UNUSED_RESULT zext(unsigned width) const
Zero extend to a new width.
Legacy analysis pass which computes a DominatorTree.
const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty)
getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the input value to the speci...
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
void setIncomingValue(unsigned i, Value *V)
bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE)
Return true if the given expression is safe to expand in the sense that all materialized values are s...
0 0 1 1 True if ordered and greater than or equal
static bool hasConcreteDefImpl(Value *V, SmallPtrSetImpl< Value * > &Visited, unsigned Depth)
Recursive helper for hasConcreteDef().
int getBasicBlockIndex(const BasicBlock *BB) const
getBasicBlockIndex - Return the first index of the specified basic block in the value list for this P...
const BasicBlock * getParent() const
Pass * createIndVarSimplifyPass()
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property...
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
SCEVConstant - This class represents a constant integer value.
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)
static PHINode * FindLoopCounter(Loop *L, const SCEV *BECount, ScalarEvolution *SE, DominatorTree *DT)
FindLoopCounter - Find an affine IV in canonical form.