98 #define DEBUG_TYPE "scalar-evolution"
101 "Number of trip counts computed with array length");
103 "Number of loops with predictable loop counts");
105 "Number of loops without predictable loop counts");
106 STATISTIC(NumBruteForceTripCountsComputed,
107 "Number of loops with trip counts computed by force");
111 cl::desc(
"Maximum number of iterations SCEV will "
112 "symbolically execute a constant "
119 cl::desc(
"Verify ScalarEvolution's backedge taken counts (slow)"));
122 cl::desc(
"Verify no dangling value in ScalarEvolution's "
123 "ExprValueMap (slow)"));
127 cl::desc(
"Threshold for inlining multiplication operands into a SCEV"),
131 "scalar-evolution-max-scev-compare-depth",
cl::Hidden,
132 cl::desc(
"Maximum depth of recursive SCEV complexity comparisons"),
136 "scalar-evolution-max-value-compare-depth",
cl::Hidden,
137 cl::desc(
"Maximum depth of recursive value complexity comparisons"),
157 cast<SCEVConstant>(
this)->getValue()->printAsOperand(OS,
false);
162 OS <<
"(trunc " << *Op->getType() <<
" " << *Op <<
" to "
169 OS <<
"(zext " << *Op->getType() <<
" " << *Op <<
" to "
176 OS <<
"(sext " << *Op->getType() <<
" " << *Op <<
" to "
202 const char *OpStr =
nullptr;
213 if (std::next(I) !=
E)
229 OS <<
"(" << *UDiv->
getLHS() <<
" /u " << *UDiv->
getRHS() <<
")";
236 OS <<
"sizeof(" << *AllocTy <<
")";
240 OS <<
"alignof(" << *AllocTy <<
")";
247 OS <<
"offsetof(" << *CTy <<
", ";
258 OS <<
"***COULDNOTCOMPUTE***";
267 return cast<SCEVConstant>(
this)->
getType();
271 return cast<SCEVCastExpr>(
this)->
getType();
276 return cast<SCEVNAryExpr>(
this)->
getType();
278 return cast<SCEVAddExpr>(
this)->
getType();
280 return cast<SCEVUDivExpr>(
this)->
getType();
282 return cast<SCEVUnknown>(
this)->
getType();
291 return SC->getValue()->isZero();
297 return SC->getValue()->isOne();
303 return SC->getValue()->isAllOnesValue();
309 if (!Mul)
return false;
313 if (!SC)
return false;
331 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
333 UniqueSCEVs.InsertNode(S, IP);
349 :
SCEV(ID, SCEVTy),
Op(op), Ty(ty) {}
356 "Cannot truncate non-integer value!");
364 "Cannot zero extend non-integer value!");
372 "Cannot sign extend non-integer value!");
375 void SCEVUnknown::deleted() {
377 SE->forgetMemoizedResults(
this);
380 SE->UniqueSCEVs.RemoveNode(
this);
386 void SCEVUnknown::allUsesReplacedWith(
Value *New) {
388 SE->forgetMemoizedResults(
this);
391 SE->UniqueSCEVs.RemoveNode(
this);
401 if (VCE->getOpcode() == Instruction::PtrToInt)
402 if (
ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
403 if (CE->getOpcode() == Instruction::GetElementPtr &&
404 CE->getOperand(0)->isNullValue() &&
405 CE->getNumOperands() == 2)
406 if (
ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1)))
408 AllocTy = cast<PointerType>(CE->getOperand(0)->getType())
418 if (VCE->getOpcode() == Instruction::PtrToInt)
419 if (
ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
420 if (CE->getOpcode() == Instruction::GetElementPtr &&
421 CE->getOperand(0)->isNullValue()) {
423 cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
424 if (
StructType *STy = dyn_cast<StructType>(Ty))
425 if (!STy->isPacked() &&
426 CE->getNumOperands() == 3 &&
427 CE->getOperand(1)->isNullValue()) {
428 if (
ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2)))
430 STy->getNumElements() == 2 &&
431 STy->getElementType(0)->isIntegerTy(1)) {
432 AllocTy = STy->getElementType(1);
443 if (VCE->getOpcode() == Instruction::PtrToInt)
444 if (
ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
445 if (CE->getOpcode() == Instruction::GetElementPtr &&
446 CE->getNumOperands() == 3 &&
447 CE->getOperand(0)->isNullValue() &&
448 CE->getOperand(1)->isNullValue()) {
450 cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
455 FieldNo = CE->getOperand(2);
494 bool LIsPointer = LV->getType()->isPointerTy(),
495 RIsPointer = RV->getType()->isPointerTy();
496 if (LIsPointer != RIsPointer)
497 return (
int)LIsPointer - (int)RIsPointer;
500 unsigned LID = LV->getValueID(), RID = RV->getValueID();
502 return (
int)LID - (int)RID;
505 if (
const auto *LA = dyn_cast<Argument>(LV)) {
506 const auto *RA = cast<Argument>(RV);
507 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
508 return (
int)LArgNo - (int)RArgNo;
511 if (
const auto *LGV = dyn_cast<GlobalValue>(LV)) {
512 const auto *RGV = cast<GlobalValue>(RV);
514 const auto IsGVNameSemantic = [&](
const GlobalValue *GV) {
515 auto LT = GV->getLinkage();
522 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
523 return LGV->getName().compare(RGV->getName());
528 if (
const auto *LInst = dyn_cast<Instruction>(LV)) {
529 const auto *RInst = cast<Instruction>(RV);
534 if (LParent != RParent) {
535 unsigned LDepth = LI->getLoopDepth(LParent),
536 RDepth = LI->getLoopDepth(RParent);
537 if (LDepth != RDepth)
538 return (
int)LDepth - (int)RDepth;
542 unsigned LNumOps = LInst->getNumOperands(),
543 RNumOps = RInst->getNumOperands();
544 if (LNumOps != RNumOps)
545 return (
int)LNumOps - (int)RNumOps;
547 for (
unsigned Idx :
seq(0u, LNumOps)) {
550 RInst->getOperand(Idx),
Depth + 1);
556 EqCache.insert({LV, RV});
564 SmallSet<std::pair<const SCEV *, const SCEV *>, 8> &EqCacheSCEV,
566 unsigned Depth = 0) {
574 return (
int)LType - (int)RType;
581 switch (static_cast<SCEVTypes>(LType)) {
590 EqCacheSCEV.insert({LHS, RHS});
600 const APInt &RA = RC->getAPInt();
602 if (LBitWidth != RBitWidth)
603 return (
int)LBitWidth - (int)RBitWidth;
604 return LA.ult(RA) ? -1 : 1;
612 const Loop *LLoop = LA->
getLoop(), *RLoop = RA->getLoop();
613 if (LLoop != RLoop) {
614 unsigned LDepth = LLoop->getLoopDepth(), RDepth = RLoop->getLoopDepth();
615 if (LDepth != RDepth)
616 return (
int)LDepth - (int)RDepth;
620 unsigned LNumOps = LA->
getNumOperands(), RNumOps = RA->getNumOperands();
621 if (LNumOps != RNumOps)
622 return (
int)LNumOps - (int)RNumOps;
625 for (
unsigned i = 0;
i != LNumOps; ++
i) {
627 RA->getOperand(
i),
Depth + 1);
631 EqCacheSCEV.insert({LHS, RHS});
643 unsigned LNumOps = LC->
getNumOperands(), RNumOps = RC->getNumOperands();
644 if (LNumOps != RNumOps)
645 return (
int)LNumOps - (int)RNumOps;
647 for (
unsigned i = 0;
i != LNumOps; ++
i) {
651 RC->getOperand(
i),
Depth + 1);
655 EqCacheSCEV.insert({LHS, RHS});
671 EqCacheSCEV.insert({LHS, RHS});
683 RC->getOperand(),
Depth + 1);
685 EqCacheSCEV.insert({LHS, RHS});
707 if (Ops.
size() < 2)
return;
710 if (Ops.
size() == 2) {
713 const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
720 std::stable_sort(Ops.
begin(), Ops.
end(),
721 [&EqCache, LI](
const SCEV *LHS,
const SCEV *RHS) {
729 for (
unsigned i = 0, e = Ops.
size();
i != e-2; ++
i) {
730 const SCEV *S = Ops[
i];
735 for (
unsigned j =
i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
740 if (
i == e-2)
return;
748 struct FindSCEVSize {
750 FindSCEVSize() : Size(0) {}
752 bool follow(
const SCEV *S) {
757 bool isDone()
const {
770 struct SCEVDivision :
public SCEVVisitor<SCEVDivision, void> {
775 const SCEV *Denominator,
const SCEV **Quotient,
776 const SCEV **Remainder) {
777 assert(Numerator && Denominator &&
"Uninitialized SCEV");
779 SCEVDivision
D(SE, Numerator, Denominator);
783 if (Numerator == Denominator) {
789 if (Numerator->
isZero()) {
796 if (Denominator->
isOne()) {
797 *Quotient = Numerator;
803 if (
const SCEVMulExpr *
T = dyn_cast<SCEVMulExpr>(Denominator)) {
805 *Quotient = Numerator;
806 for (
const SCEV *
Op :
T->operands()) {
807 divide(SE, *Quotient,
Op, &Q, &R);
814 *Remainder = Numerator;
823 *Quotient =
D.Quotient;
824 *Remainder =
D.Remainder;
839 if (
const SCEVConstant *
D = dyn_cast<SCEVConstant>(Denominator)) {
841 APInt DenominatorVal =
D->getAPInt();
845 if (NumeratorBW > DenominatorBW)
846 DenominatorVal = DenominatorVal.
sext(NumeratorBW);
847 else if (NumeratorBW < DenominatorBW)
848 NumeratorVal = NumeratorVal.
sext(DenominatorBW);
852 APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal);
860 const SCEV *StartQ, *StartR, *StepQ, *StepR;
862 return cannotDivide(Numerator);
863 divide(SE, Numerator->
getStart(), Denominator, &StartQ, &StartR);
867 if (Ty != StartQ->getType() || Ty != StartR->
getType() ||
868 Ty != StepQ->getType() || Ty != StepR->
getType())
869 return cannotDivide(Numerator);
882 divide(SE,
Op, Denominator, &Q, &R);
886 return cannotDivide(Numerator);
892 if (Qs.
size() == 1) {
906 bool FoundDenominatorTerm =
false;
909 if (Ty !=
Op->getType())
910 return cannotDivide(Numerator);
912 if (FoundDenominatorTerm) {
919 divide(SE,
Op, Denominator, &Q, &R);
927 return cannotDivide(Numerator);
929 FoundDenominatorTerm =
true;
933 if (FoundDenominatorTerm) {
942 if (!isa<SCEVUnknown>(Denominator))
943 return cannotDivide(Numerator);
947 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
948 cast<SCEVConstant>(
Zero)->getValue();
951 if (Remainder->
isZero()) {
953 RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
954 cast<SCEVConstant>(One)->getValue();
965 return cannotDivide(Numerator);
966 divide(SE, Diff, Denominator, &Q, &R);
968 return cannotDivide(Numerator);
974 const SCEV *Denominator)
975 : SE(S), Denominator(Denominator) {
982 cannotDivide(Numerator);
987 void cannotDivide(
const SCEV *Numerator) {
989 Remainder = Numerator;
993 const SCEV *Denominator, *Quotient, *Remainder, *
Zero, *One;
1070 APInt OddFactorial(W, 1);
1072 for (
unsigned i = 3;
i <= K; ++
i) {
1076 Mult = Mult.
lshr(TwoFactors);
1077 OddFactorial *=
Mult;
1081 unsigned CalculationBits = W +
T;
1084 APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
1089 APInt Mod = APInt::getSignedMinValue(W+1);
1090 APInt MultiplyFactor = OddFactorial.
zext(W+1);
1092 MultiplyFactor = MultiplyFactor.
trunc(W);
1098 for (
unsigned i = 1;
i != K; ++
i) {
1122 const SCEV *SCEVAddRecExpr::evaluateAtIteration(
const SCEV *It,
1130 if (isa<SCEVCouldNotCompute>(Coeff))
1145 "This is not a truncating conversion!");
1147 "This is not a conversion to a SCEVable type!");
1155 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
1176 if (
const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) {
1178 bool hasTrunc =
false;
1179 for (
unsigned i = 0, e = SA->getNumOperands();
i != e && !hasTrunc; ++
i) {
1181 if (!isa<SCEVCastExpr>(SA->getOperand(
i)))
1182 hasTrunc = isa<SCEVTruncateExpr>(S);
1187 UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
1192 if (
const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) {
1194 bool hasTrunc =
false;
1195 for (
unsigned i = 0, e = SM->getNumOperands();
i != e && !hasTrunc; ++
i) {
1197 if (!isa<SCEVCastExpr>(SM->getOperand(
i)))
1198 hasTrunc = isa<SCEVTruncateExpr>(S);
1203 UniqueSCEVs.FindNodeOrInsertPos(ID, IP);
1207 if (
const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
1209 for (
const SCEV *Op : AddRec->operands())
1219 UniqueSCEVs.InsertNode(S, IP);
1258 struct ExtendOpTraitsBase {
1263 template <
typename ExtendOp>
struct ExtendOpTraits {
1279 static const GetExtendExprTy GetExtendExpr;
1281 static const SCEV *getOverflowLimitForStep(
const SCEV *Step,
1288 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1295 static const GetExtendExprTy GetExtendExpr;
1297 static const SCEV *getOverflowLimitForStep(
const SCEV *Step,
1304 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1315 template <
typename ExtendOpTy>
1318 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1319 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1345 auto PreStartFlags =
1363 const SCEV *OperandExtendedStart =
1364 SE->
getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy),
1365 (SE->*GetExtendExpr)(Step, WideTy));
1366 if ((SE->*GetExtendExpr)(Start, WideTy) == OperandExtendedStart) {
1378 const SCEV *OverflowLimit =
1379 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1381 if (OverflowLimit &&
1389 template <
typename ExtendOpTy>
1392 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1394 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE);
1396 return (SE->*GetExtendExpr)(AR->getStart(), Ty);
1398 return SE->
getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty),
1399 (SE->*GetExtendExpr)(PreStart, Ty));
1435 template <
typename ExtendOpTy>
1436 bool ScalarEvolution::proveNoWrapByVaryingStart(
const SCEV *Start,
1439 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1452 for (
unsigned Delta : {-2, -1, 1, 2}) {
1462 static_cast<SCEVAddRecExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
1466 if (PreAR && PreAR->getNoWrapFlags(WrapType)) {
1469 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1470 DeltaS, &Pred,
this);
1482 "This is not an extending conversion!");
1484 "This is not a conversion to a SCEVable type!");
1503 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
1509 const SCEV *X =
ST->getOperand();
1523 if (AR->isAffine()) {
1524 const SCEV *Start = AR->getStart();
1525 const SCEV *Step = AR->getStepRecurrence(*
this);
1527 const Loop *L = AR->getLoop();
1529 if (!AR->hasNoUnsignedWrap()) {
1530 auto NewFlags = proveNoWrapViaConstantRanges(AR);
1536 if (AR->hasNoUnsignedWrap())
1538 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty,
this),
1550 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
1556 const SCEV *CastedMaxBECount =
1558 const SCEV *RecastedMaxBECount =
1560 if (MaxBECount == RecastedMaxBECount) {
1566 const SCEV *WideMaxBECount =
1568 const SCEV *OperandExtendedAdd =
1572 if (ZAdd == OperandExtendedAdd) {
1577 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty,
this),
1582 OperandExtendedAdd =
1586 if (ZAdd == OperandExtendedAdd) {
1592 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty,
this),
1605 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
1618 AR->getPostIncExpr(*
this),
N))) {
1624 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty,
this),
1633 AR->getPostIncExpr(*
this),
N))) {
1640 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty,
this),
1646 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1649 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty,
this),
1654 if (
auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1656 if (SA->hasNoUnsignedWrap()) {
1660 for (
const auto *Op : SA->operands())
1668 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
1671 UniqueSCEVs.InsertNode(S, IP);
1678 "This is not an extending conversion!");
1680 "This is not a conversion to a SCEVable type!");
1703 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
1709 const SCEV *X =
ST->getOperand();
1719 if (
auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
1720 if (SA->getNumOperands() == 2) {
1724 if (
auto *SC2 = dyn_cast<SCEVConstant>(SMul->getOperand(0))) {
1725 const APInt &C1 = SC1->getAPInt();
1726 const APInt &C2 = SC2->getAPInt();
1736 if (SA->hasNoSignedWrap()) {
1740 for (
const auto *Op : SA->operands())
1750 if (AR->isAffine()) {
1751 const SCEV *Start = AR->getStart();
1752 const SCEV *Step = AR->getStepRecurrence(*
this);
1754 const Loop *L = AR->getLoop();
1756 if (!AR->hasNoSignedWrap()) {
1757 auto NewFlags = proveNoWrapViaConstantRanges(AR);
1763 if (AR->hasNoSignedWrap())
1765 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty,
this),
1777 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
1783 const SCEV *CastedMaxBECount =
1785 const SCEV *RecastedMaxBECount =
1787 if (MaxBECount == RecastedMaxBECount) {
1793 const SCEV *WideMaxBECount =
1795 const SCEV *OperandExtendedAdd =
1799 if (SAdd == OperandExtendedAdd) {
1804 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty,
this),
1809 OperandExtendedAdd =
1813 if (SAdd == OperandExtendedAdd) {
1826 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty,
this),
1840 if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
1848 const SCEV *OverflowLimit =
1850 if (OverflowLimit &&
1858 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty,
this),
1869 const APInt &C1 = SC1->getAPInt();
1870 const APInt &C2 = SC2->getAPInt();
1875 AR->getNoWrapFlags());
1880 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
1883 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty,
this),
1895 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
1898 UniqueSCEVs.InsertNode(S, IP);
1908 "This is not an extending conversion!");
1910 "This is not a conversion to a SCEVable type!");
1915 if (
SC->getAPInt().isNegative())
1920 const SCEV *NewOp =
T->getOperand();
1928 if (!isa<SCEVZeroExtendExpr>(ZExt))
1933 if (!isa<SCEVSignExtendExpr>(SExt))
1939 for (
const SCEV *Op : AR->operands())
1945 if (isa<SCEVSMaxExpr>(Op))
1979 APInt &AccumulatedConstant,
1980 const SCEV *
const *Ops,
size_t NumOperands,
1983 bool Interesting =
false;
1987 while (
const SCEVConstant *
C = dyn_cast<SCEVConstant>(Ops[i])) {
1990 if (Scale != 1 || AccumulatedConstant != 0 ||
C->getValue()->isZero())
1992 AccumulatedConstant += Scale *
C->getAPInt();
1997 for (; i != NumOperands; ++
i) {
1999 if (Mul && isa<SCEVConstant>(Mul->
getOperand(0))) {
2001 Scale * cast<SCEVConstant>(Mul->
getOperand(0))->getAPInt();
2014 auto Pair = M.
insert({Key, NewScale});
2018 Pair.first->second += NewScale;
2026 std::pair<DenseMap<const SCEV *, APInt>::iterator,
bool> Pair =
2031 Pair.first->second += Scale;
2049 using namespace std::placeholders;
2055 assert(CanAnalyze &&
"don't call from other places!");
2062 auto IsKnownNonNegative = [&](
const SCEV *S) {
2072 if (SignOrUnsignWrap != SignOrUnsignMask && Type ==
scAddExpr &&
2073 Ops.
size() == 2 && isa<SCEVConstant>(Ops[0])) {
2078 const APInt &
C = cast<SCEVConstant>(Ops[0])->getAPInt();
2100 "only nuw or nsw allowed");
2102 if (Ops.
size() == 1)
return Ops[0];
2105 for (
unsigned i = 1, e = Ops.
size();
i != e; ++
i)
2107 "SCEVAddExpr operand types don't match!");
2117 if (
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
2120 while (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
2122 Ops[0] =
getConstant(LHSC->getAPInt() + RHSC->getAPInt());
2123 if (Ops.
size() == 2)
return Ops[0];
2125 LHSC = cast<SCEVConstant>(Ops[0]);
2129 if (LHSC->getValue()->isZero()) {
2134 if (Ops.
size() == 1)
return Ops[0];
2140 Type *Ty = Ops[0]->getType();
2141 bool FoundMatch =
false;
2142 for (
unsigned i = 0, e = Ops.
size();
i != e-1; ++
i)
2143 if (Ops[
i] == Ops[
i+1]) {
2146 while (
i+Count != e && Ops[
i+Count] == Ops[
i])
2151 if (Ops.
size() == Count)
2155 --
i; e -= Count - 1;
2165 for (; Idx < Ops.
size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) {
2173 for (
unsigned i = 0, e = Ops.
size();
i != e; ++
i) {
2175 if (
T->getOperand()->getType() != SrcType) {
2180 }
else if (
const SCEVConstant *
C = dyn_cast<SCEVConstant>(Ops[i])) {
2182 }
else if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) {
2184 for (
unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2186 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
2187 if (
T->getOperand()->getType() != SrcType) {
2192 }
else if (
const auto *
C = dyn_cast<SCEVConstant>(M->getOperand(j))) {
2210 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
2216 while (Idx < Ops.
size() && Ops[Idx]->getSCEVType() <
scAddExpr)
2220 if (Idx < Ops.
size()) {
2221 bool DeletedAdd =
false;
2222 while (
const SCEVAddExpr *
Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
2238 while (Idx < Ops.
size() && Ops[Idx]->getSCEVType() <
scMulExpr)
2243 if (Idx < Ops.
size() && isa<SCEVMulExpr>(Ops[Idx])) {
2247 APInt AccumulatedConstant(BitWidth, 0);
2250 APInt(BitWidth, 1), *
this)) {
2251 struct APIntCompare {
2252 bool operator()(
const APInt &LHS,
const APInt &RHS)
const {
2253 return LHS.
ult(RHS);
2260 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
2261 for (
const SCEV *NewOp : NewOps)
2262 MulOpLists[M.
find(NewOp)->second].push_back(NewOp);
2265 if (AccumulatedConstant != 0)
2267 for (
auto &MulOp : MulOpLists)
2268 if (MulOp.first != 0)
2273 if (Ops.
size() == 1)
2282 for (; Idx < Ops.
size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
2283 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
2284 for (
unsigned MulOp = 0, e = Mul->
getNumOperands(); MulOp != e; ++MulOp) {
2286 if (isa<SCEVConstant>(MulOpSCEV))
2288 for (
unsigned AddOp = 0, e = Ops.
size(); AddOp != e; ++AddOp)
2289 if (MulOpSCEV == Ops[AddOp]) {
2303 if (Ops.
size() == 2)
return OuterMul;
2316 for (
unsigned OtherMulIdx = Idx+1;
2317 OtherMulIdx < Ops.
size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
2319 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
2323 OMulOp != e; ++OMulOp)
2324 if (OtherMul->
getOperand(OMulOp) == MulOpSCEV) {
2342 if (Ops.
size() == 2)
return OuterMul;
2359 for (; Idx < Ops.
size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
2365 for (
unsigned i = 0, e = Ops.
size();
i != e; ++
i)
2373 if (!LIOps.
empty()) {
2391 if (Ops.
size() == 1)
return NewRec;
2394 for (
unsigned i = 0;; ++
i)
2395 if (Ops[i] == AddRec) {
2405 for (
unsigned OtherIdx = Idx+1;
2406 OtherIdx < Ops.
size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2408 if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
2412 for (; OtherIdx != Ops.
size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2414 if (
const auto *OtherAddRec = dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]))
2415 if (OtherAddRec->getLoop() == AddRecLoop) {
2416 for (
unsigned i = 0, e = OtherAddRec->getNumOperands();
2418 if (i >= AddRecOps.size()) {
2419 AddRecOps.
append(OtherAddRec->op_begin()+
i,
2420 OtherAddRec->op_end());
2424 OtherAddRec->getOperand(i));
2426 Ops.
erase(Ops.
begin() + OtherIdx); --OtherIdx;
2441 for (
unsigned i = 0, e = Ops.
size();
i != e; ++
i)
2445 static_cast<SCEVAddExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2448 std::uninitialized_copy(Ops.
begin(), Ops.
end(), O);
2451 UniqueSCEVs.InsertNode(S, IP);
2457 static uint64_t
umul_ov(uint64_t
i, uint64_t j,
bool &Overflow) {
2459 if (j > 1 && k / j != i) Overflow =
true;
2466 static uint64_t
Choose(uint64_t n, uint64_t k,
bool &Overflow) {
2475 if (n == 0 || n == k)
return 1;
2476 if (k > n)
return 0;
2482 for (uint64_t
i = 1;
i <= k; ++
i) {
2494 while (!Ops.
empty()) {
2496 if (isa<SCEVConstant>(*CurrentExpr))
2499 if (isa<SCEVAddExpr>(*CurrentExpr) || isa<SCEVMulExpr>(*CurrentExpr)) {
2500 const auto *CurrentNAry = cast<SCEVNAryExpr>(CurrentExpr);
2501 Ops.
append(CurrentNAry->op_begin(), CurrentNAry->op_end());
2511 "only nuw or nsw allowed");
2513 if (Ops.
size() == 1)
return Ops[0];
2516 for (
unsigned i = 1, e = Ops.
size();
i != e; ++
i)
2518 "SCEVMulExpr operand types don't match!");
2528 if (
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
2531 if (Ops.
size() == 2)
2535 if (
Add->getNumOperands() == 2)
2541 while (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
2546 Ops.erase(Ops.begin()+1);
2547 if (Ops.size() == 1)
return Ops[0];
2548 LHSC = cast<SCEVConstant>(Ops[0]);
2552 if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) {
2555 }
else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) {
2558 }
else if (Ops[0]->isAllOnesValue()) {
2561 if (Ops.
size() == 2) {
2564 bool AnyFolded =
false;
2565 for (
const SCEV *AddOp :
Add->operands()) {
2567 if (!isa<SCEVMulExpr>(Mul)) AnyFolded =
true;
2572 }
else if (
const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
2575 for (
const SCEV *AddRecOp : AddRec->operands())
2584 if (Ops.
size() == 1)
2589 while (Idx < Ops.
size() && Ops[Idx]->getSCEVType() <
scMulExpr)
2593 if (Idx < Ops.
size()) {
2594 bool DeletedMul =
false;
2595 while (
const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
2601 Ops.
append(Mul->op_begin(), Mul->op_end());
2619 for (; Idx < Ops.
size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
2625 for (
unsigned i = 0, e = Ops.
size();
i != e; ++
i)
2633 if (!LIOps.
empty()) {
2650 if (Ops.
size() == 1)
return NewRec;
2653 for (
unsigned i = 0;; ++
i)
2654 if (Ops[i] == AddRec) {
2675 bool OpsModified =
false;
2676 for (
unsigned OtherIdx = Idx+1;
2677 OtherIdx != Ops.
size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
2681 if (!OtherAddRec || OtherAddRec->
getLoop() != AddRecLoop)
2684 bool Overflow =
false;
2691 for (
int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
2692 uint64_t Coeff1 =
Choose(x, 2*x - y, Overflow);
2695 z < ze && !Overflow; ++z) {
2696 uint64_t Coeff2 =
Choose(2*x - y, x-z, Overflow);
2698 if (LargerThan64Bits)
2699 Coeff =
umul_ov(Coeff1, Coeff2, Overflow);
2701 Coeff = Coeff1*Coeff2;
2713 if (Ops.
size() == 2)
return NewAddRec;
2714 Ops[Idx] = NewAddRec;
2715 Ops.
erase(Ops.
begin() + OtherIdx); --OtherIdx;
2733 for (
unsigned i = 0, e = Ops.
size();
i != e; ++
i)
2737 static_cast<SCEVMulExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2740 std::uninitialized_copy(Ops.
begin(), Ops.
end(), O);
2743 UniqueSCEVs.InsertNode(S, IP);
2755 "SCEVUDivExpr operand types don't match!");
2757 if (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
2758 if (RHSC->getValue()->equalsInt(1))
2763 if (!RHSC->getValue()->isZero()) {
2768 unsigned LZ = RHSC->getAPInt().countLeadingZeros();
2772 if (!RHSC->getAPInt().isPowerOf2())
2778 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*
this))) {
2780 const APInt &StepInt = Step->getAPInt();
2781 const APInt &DivInt = RHSC->getAPInt();
2782 if (!StepInt.
urem(DivInt) &&
2788 for (
const SCEV *
Op : AR->operands())
2796 if (StartC && !DivInt.
urem(StepInt) &&
2802 const APInt &StartRem = StartInt.
urem(StepInt);
2809 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
2811 for (
const SCEV *
Op : M->operands())
2815 for (
unsigned i = 0, e = M->getNumOperands();
i != e; ++
i) {
2816 const SCEV *
Op = M->getOperand(
i);
2818 if (!isa<SCEVUDivExpr>(Div) &&
getMulExpr(Div, RHSC) == Op) {
2827 if (
const SCEVAddExpr *
A = dyn_cast<SCEVAddExpr>(LHS)) {
2829 for (
const SCEV *
Op :
A->operands())
2833 for (
unsigned i = 0, e =
A->getNumOperands();
i != e; ++
i) {
2835 if (isa<SCEVUDivExpr>(Op) ||
2840 if (Operands.
size() ==
A->getNumOperands())
2846 if (
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
2847 Constant *LHSCV = LHSC->getValue();
2848 Constant *RHSCV = RHSC->getValue();
2860 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
2863 UniqueSCEVs.InsertNode(S, IP);
2895 if (
const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
2898 if (
const auto *LHSCst = dyn_cast<SCEVConstant>(Mul->
getOperand(0))) {
2899 if (LHSCst == RHSCst) {
2908 APInt Factor =
gcd(LHSCst, RHSCst);
2911 cast<SCEVConstant>(
getConstant(LHSCst->getAPInt().udiv(Factor)));
2913 cast<SCEVConstant>(
getConstant(RHSCst->getAPInt().udiv(Factor)));
2945 if (
const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
2946 if (StepChrec->getLoop() ==
L) {
2947 Operands.
append(StepChrec->op_begin(), StepChrec->op_end());
2960 if (Operands.
size() == 1)
return Operands[0];
2963 for (
unsigned i = 1, e = Operands.
size();
i != e; ++
i)
2965 "SCEVAddRecExpr operand types don't match!");
2966 for (
unsigned i = 0, e = Operands.
size();
i != e; ++
i)
2968 "SCEVAddRecExpr operand is not loop-invariant!");
2985 if (
const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
2986 const Loop *NestedLoop = NestedAR->getLoop();
2992 NestedAR->op_end());
2993 Operands[0] = NestedAR->getStart();
2997 bool AllInvariant =
all_of(
3009 AllInvariant =
all_of(NestedOperands, [&](
const SCEV *Op) {
3020 return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
3024 Operands[0] = NestedAR;
3032 for (
unsigned i = 0, e = Operands.
size(); i != e; ++
i)
3037 static_cast<SCEVAddRecExpr *
>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
3040 std::uninitialized_copy(Operands.
begin(), Operands.
end(), O);
3042 O, Operands.
size(),
L);
3043 UniqueSCEVs.InsertNode(S, IP);
3068 for (
const SCEV *IndexExpr : IndexExprs) {
3070 if (
StructType *STy = dyn_cast<StructType>(CurTy)) {
3072 ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue();
3077 TotalOffset =
getAddExpr(TotalOffset, FieldOffset);
3080 CurTy = STy->getTypeAtIndex(Index);
3083 CurTy = cast<SequentialType>(CurTy)->getElementType();
3090 const SCEV *LocalOffset =
getMulExpr(IndexExpr, ElementSize, Wrap);
3093 TotalOffset =
getAddExpr(TotalOffset, LocalOffset);
3098 return getAddExpr(BaseExpr, TotalOffset, Wrap);
3110 if (Ops.
size() == 1)
return Ops[0];
3113 for (
unsigned i = 1, e = Ops.
size();
i != e; ++
i)
3115 "SCEVSMaxExpr operand types don't match!");
3123 if (
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3126 while (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
3132 if (Ops.
size() == 1)
return Ops[0];
3133 LHSC = cast<SCEVConstant>(Ops[0]);
3137 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(
true)) {
3140 }
else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(
true)) {
3146 if (Ops.
size() == 1)
return Ops[0];
3155 if (Idx < Ops.
size()) {
3156 bool DeletedSMax =
false;
3157 while (
const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) {
3159 Ops.
append(SMax->op_begin(), SMax->op_end());
3170 for (
unsigned i = 0, e = Ops.
size()-1;
i != e; ++
i)
3173 if (Ops[
i] == Ops[
i+1] ||
3182 if (Ops.
size() == 1)
return Ops[0];
3184 assert(!Ops.
empty() &&
"Reduced smax down to nothing!");
3190 for (
unsigned i = 0, e = Ops.
size(); i != e; ++
i)
3191 ID.AddPointer(Ops[i]);
3193 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
3195 std::uninitialized_copy(Ops.
begin(), Ops.
end(), O);
3198 UniqueSCEVs.InsertNode(S, IP);
3211 if (Ops.
size() == 1)
return Ops[0];
3214 for (
unsigned i = 1, e = Ops.
size();
i != e; ++
i)
3216 "SCEVUMaxExpr operand types don't match!");
3224 if (
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
3227 while (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
3233 if (Ops.
size() == 1)
return Ops[0];
3234 LHSC = cast<SCEVConstant>(Ops[0]);
3238 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(
false)) {
3241 }
else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(
false)) {
3247 if (Ops.
size() == 1)
return Ops[0];
3256 if (Idx < Ops.
size()) {
3257 bool DeletedUMax =
false;
3258 while (
const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) {
3260 Ops.
append(UMax->op_begin(), UMax->op_end());
3271 for (
unsigned i = 0, e = Ops.
size()-1;
i != e; ++
i)
3274 if (Ops[
i] == Ops[
i+1] ||
3283 if (Ops.
size() == 1)
return Ops[0];
3285 assert(!Ops.
empty() &&
"Reduced umax down to nothing!");
3291 for (
unsigned i = 0, e = Ops.
size(); i != e; ++
i)
3292 ID.AddPointer(Ops[i]);
3294 if (
const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP))
return S;
3296 std::uninitialized_copy(Ops.
begin(), Ops.
end(), O);
3299 UniqueSCEVs.InsertNode(S, IP);
3329 IntTy,
getDataLayout().getStructLayout(STy)->getElementOffset(FieldNo));
3342 if (
SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
3343 assert(cast<SCEVUnknown>(S)->getValue() == V &&
3344 "Stale SCEVUnknown in uniquing map!");
3349 FirstUnknown = cast<SCEVUnknown>(S);
3350 UniqueSCEVs.InsertNode(S, IP);
3389 return CouldNotCompute.get();
3392 bool ScalarEvolution::checkValidity(
const SCEV *S)
const {
3395 return SU && SU->
getValue() ==
nullptr;
3398 return !ContainsNulls;
3403 if (I != HasRecMap.
end())
3407 HasRecMap.
insert({S, FoundAddRec});
3417 return {S,
nullptr};
3419 if (
Add->getNumOperands() != 2)
3420 return {S,
nullptr};
3424 return {S,
nullptr};
3426 return {
Add->getOperand(1), ConstOp->getValue()};
3434 if (SI == ExprValueMap.
end())
3439 for (
const auto &VE : SI->second)
3451 if (I != ValueExprMap.
end()) {
3452 const SCEV *S = I->second;
3455 SV->remove({V,
nullptr});
3458 const SCEV *Stripped;
3461 if (Offset !=
nullptr) {
3463 SV->remove({V, Offset});
3465 ValueExprMap.
erase(V);
3474 const SCEV *S = getExistingSCEV(V);
3480 std::pair<ValueExprMapType::iterator, bool> Pair =
3483 ExprValueMap[S].
insert({V,
nullptr});
3487 const SCEV *Stripped = S;
3495 if (Offset !=
nullptr && !isa<SCEVUnknown>(Stripped) &&
3496 !isa<GetElementPtrInst>(V))
3497 ExprValueMap[Stripped].
insert({V, Offset});
3503 const SCEV *ScalarEvolution::getExistingSCEV(
Value *V) {
3507 if (I != ValueExprMap.
end()) {
3508 const SCEV *S = I->second;
3509 if (checkValidity(S))
3512 forgetMemoizedResults(S);
3539 const SCEV *AllOnes =
3553 const bool RHSIsNotMinSigned =
3587 "Cannot truncate or zero extend with non-integer arguments!");
3601 "Cannot truncate or zero extend with non-integer arguments!");
3614 "Cannot noop or zero extend with non-integer arguments!");
3616 "getNoopOrZeroExtend cannot truncate!");
3627 "Cannot noop or sign extend with non-integer arguments!");
3629 "getNoopOrSignExtend cannot truncate!");
3640 "Cannot noop or any extend with non-integer arguments!");
3642 "getNoopOrAnyExtend cannot truncate!");
3653 "Cannot truncate or noop with non-integer arguments!");
3655 "getTruncateOrNoop cannot extend!");
3663 const SCEV *PromotedLHS = LHS;
3664 const SCEV *PromotedRHS = RHS;
3676 const SCEV *PromotedLHS = LHS;
3677 const SCEV *PromotedRHS = RHS;
3692 if (
const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
3694 }
else if (
const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) {
3695 const SCEV *PtrOp =
nullptr;
3696 for (
const SCEV *NAryOp : NAry->operands()) {
3697 if (NAryOp->getType()->isPointerTy()) {
3717 Worklist.
push_back(cast<Instruction>(U));
3720 void ScalarEvolution::forgetSymbolicName(
Instruction *PN,
const SCEV *SymName) {
3726 while (!Worklist.
empty()) {
3728 if (!Visited.
insert(I).second)
3731 auto It = ValueExprMap.
find_as(static_cast<Value *>(I));
3732 if (It != ValueExprMap.
end()) {
3733 const SCEV *Old = It->second;
3737 if (Old != SymName && !
hasOperand(Old, SymName))
3747 if (!isa<PHINode>(I) ||
3748 !isa<SCEVUnknown>(Old) ||
3749 (I != PN && Old == SymName)) {
3751 forgetMemoizedResults(Old);
3762 static const SCEV *rewrite(
const SCEV *S,
const Loop *L,
3786 bool isValid() {
return Valid; }
3795 static const SCEV *rewrite(
const SCEV *S,
const Loop *L,
3818 bool isValid() {
return Valid; }
3827 ScalarEvolution::proveNoWrapViaConstantRanges(
const SCEVAddRecExpr *AR) {
3840 if (NSWRegion.contains(AddRecRange))
3850 if (NUWRegion.contains(AddRecRange))
3873 : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)),
3875 if (
auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) {
3876 IsNSW = OBO->hasNoSignedWrap();
3877 IsNUW = OBO->hasNoUnsignedWrap();
3881 explicit BinaryOp(
unsigned Opcode,
Value *LHS,
Value *RHS,
bool IsNSW =
false,
3883 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW),
3901 case Instruction::Sub:
3902 case Instruction::Mul:
3903 case Instruction::UDiv:
3906 case Instruction::AShr:
3907 case Instruction::Shl:
3908 return BinaryOp(Op);
3911 if (
auto *RHSC = dyn_cast<ConstantInt>(Op->
getOperand(1)))
3914 if (RHSC->getValue().isSignBit())
3916 return BinaryOp(Op);
3918 case Instruction::LShr:
3927 if (SA->getValue().ult(BitWidth)) {
3931 return BinaryOp(Instruction::UDiv, Op->
getOperand(0),
X);
3934 return BinaryOp(Op);
3936 case Instruction::ExtractValue: {
3937 auto *EVI = cast<ExtractValueInst>(
Op);
3938 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
3945 if (
auto *
F = CI->getCalledFunction())
3946 switch (
F->getIntrinsicID()) {
3947 case Intrinsic::sadd_with_overflow:
3948 case Intrinsic::uadd_with_overflow: {
3951 CI->getArgOperand(1));
3956 if (
F->getIntrinsicID() == Intrinsic::sadd_with_overflow)
3958 CI->getArgOperand(1),
true,
3962 CI->getArgOperand(1),
false,
3966 case Intrinsic::ssub_with_overflow:
3967 case Intrinsic::usub_with_overflow:
3968 return BinaryOp(Instruction::Sub, CI->getArgOperand(0),
3969 CI->getArgOperand(1));
3971 case Intrinsic::smul_with_overflow:
3972 case Intrinsic::umul_with_overflow:
3973 return BinaryOp(Instruction::Mul, CI->getArgOperand(0),
3974 CI->getArgOperand(1));
3987 const SCEV *ScalarEvolution::createAddRecFromPHI(
PHINode *PN) {
3995 Value *BEValueV =
nullptr, *StartValueV =
nullptr;
4001 }
else if (BEValueV != V) {
4005 }
else if (!StartValueV) {
4007 }
else if (StartValueV != V) {
4008 StartValueV =
nullptr;
4012 if (BEValueV && StartValueV) {
4016 "PHI node already processed?");
4031 unsigned FoundIndex =
Add->getNumOperands();
4032 for (
unsigned i = 0, e =
Add->getNumOperands();
i != e; ++
i)
4033 if (
Add->getOperand(
i) == SymbolicName)
4034 if (FoundIndex == e) {
4039 if (FoundIndex !=
Add->getNumOperands()) {
4042 for (
unsigned i = 0, e =
Add->getNumOperands();
i != e; ++
i)
4043 if (
i != FoundIndex)
4050 (isa<SCEVAddRecExpr>(Accum) &&
4051 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
4068 if (
GEP->isInBounds() &&
GEP->getOperand(0) == PN) {
4087 forgetSymbolicName(PN, SymbolicName);
4093 if (
auto *BEInst = dyn_cast<Instruction>(BEValueV))
4110 const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *
this);
4111 const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *
this);
4115 if (Start == StartVal) {
4119 forgetSymbolicName(PN, SymbolicName);
4140 struct CheckAvailable {
4141 bool TraversalDone =
false;
4142 bool Available =
true;
4144 const Loop *L =
nullptr;
4149 :
L(L), BB(BB), DT(DT) {}
4151 bool setUnavailable() {
4152 TraversalDone =
true;
4157 bool follow(
const SCEV *S) {
4170 const auto *ARLoop = cast<SCEVAddRecExpr>(S)->getLoop();
4171 if (L && (ARLoop == L || ARLoop->
contains(L)))
4174 return setUnavailable();
4179 const auto *SU = cast<SCEVUnknown>(S);
4180 Value *V = SU->getValue();
4182 if (isa<Argument>(V))
4185 if (isa<Instruction>(V) && DT.
dominates(cast<Instruction>(V), BB))
4188 return setUnavailable();
4194 return setUnavailable();
4199 bool isDone() {
return TraversalDone; }
4202 CheckAvailable CA(L, BB, DT);
4206 return CA.Available;
4219 if (!LeftEdge.isSingleEdge())
4222 assert(RightEdge.isSingleEdge() &&
"Follows from LeftEdge.isSingleEdge()");
4242 const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(
PHINode *PN) {
4266 assert(IDom &&
"At least the entry block should dominate PN");
4269 Value *Cond =
nullptr, *LHS =
nullptr, *RHS =
nullptr;
4271 if (BI && BI->isConditional() &&
4275 return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
4281 const SCEV *ScalarEvolution::createNodeForPHI(
PHINode *PN) {
4282 if (
const SCEV *S = createAddRecFromPHI(PN))
4285 if (
const SCEV *S = createNodeFromSelectLikePHI(PN))
4306 if (
auto *CI = dyn_cast<ConstantInt>(Cond))
4307 return getSCEV(CI->isOne() ? TrueVal : FalseVal);
4314 Value *LHS = ICI->getOperand(0);
4315 Value *RHS = ICI->getOperand(1);
4317 switch (ICI->getPredicate()) {
4367 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->
isZero()) {
4381 isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->
isZero()) {
4415 return C->getAPInt().countTrailingZeros();
4436 for (
unsigned i = 1, e =
A->getNumOperands(); MinOpRes &&
i != e; ++
i)
4441 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
4445 for (
unsigned i = 1, e = M->getNumOperands();
4446 SumOpRes != BitWidth &&
i != e; ++
i)
4455 for (
unsigned i = 1, e =
A->getNumOperands(); MinOpRes &&
i != e; ++
i)
4460 if (
const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) {
4463 for (
unsigned i = 1, e = M->getNumOperands(); MinOpRes &&
i != e; ++
i)
4468 if (
const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) {
4471 for (
unsigned i = 1, e = M->getNumOperands(); MinOpRes &&
i != e; ++
i)
4476 if (
const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
4479 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
4482 return Zeros.countTrailingOnes();
4502 ScalarEvolution::getRange(
const SCEV *S,
4503 ScalarEvolution::RangeSignHint SignHint) {
4505 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
4510 if (I != Cache.
end())
4523 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED)
4524 ConservativeResult =
4535 for (
unsigned i = 1, e =
Add->getNumOperands();
i != e; ++
i)
4536 X = X.
add(getRange(
Add->getOperand(
i), SignHint));
4537 return setRange(
Add, SignHint, ConservativeResult.intersectWith(X));
4540 if (
const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
4542 for (
unsigned i = 1, e = Mul->getNumOperands();
i != e; ++
i)
4543 X = X.
multiply(getRange(Mul->getOperand(
i), SignHint));
4544 return setRange(Mul, SignHint, ConservativeResult.intersectWith(X));
4547 if (
const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
4549 for (
unsigned i = 1, e = SMax->getNumOperands();
i != e; ++
i)
4550 X = X.
smax(getRange(SMax->getOperand(
i), SignHint));
4551 return setRange(SMax, SignHint, ConservativeResult.intersectWith(X));
4554 if (
const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
4556 for (
unsigned i = 1, e = UMax->getNumOperands();
i != e; ++
i)
4557 X = X.
umax(getRange(UMax->getOperand(
i), SignHint));
4558 return setRange(UMax, SignHint, ConservativeResult.intersectWith(X));
4561 if (
const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
4564 return setRange(UDiv, SignHint,
4565 ConservativeResult.intersectWith(X.
udiv(Y)));
4570 return setRange(ZExt, SignHint,
4571 ConservativeResult.intersectWith(X.
zeroExtend(BitWidth)));
4576 return setRange(SExt, SignHint,
4577 ConservativeResult.intersectWith(X.
signExtend(BitWidth)));
4582 return setRange(Trunc, SignHint,
4583 ConservativeResult.intersectWith(X.
truncate(BitWidth)));
4586 if (
const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
4589 if (AddRec->hasNoUnsignedWrap())
4590 if (
const SCEVConstant *
C = dyn_cast<SCEVConstant>(AddRec->getStart()))
4591 if (!
C->getValue()->isZero())
4592 ConservativeResult = ConservativeResult.intersectWith(
4597 if (AddRec->hasNoSignedWrap()) {
4598 bool AllNonNeg =
true;
4599 bool AllNonPos =
true;
4600 for (
unsigned i = 0, e = AddRec->getNumOperands();
i != e; ++
i) {
4605 ConservativeResult = ConservativeResult.intersectWith(
4609 ConservativeResult = ConservativeResult.intersectWith(
4611 APInt(BitWidth, 1)));
4615 if (AddRec->isAffine()) {
4617 if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
4619 auto RangeFromAffine = getRangeForAffineAR(
4620 AddRec->getStart(), AddRec->getStepRecurrence(*
this), MaxBECount,
4622 if (!RangeFromAffine.isFullSet())
4623 ConservativeResult =
4626 auto RangeFromFactoring = getRangeViaFactoring(
4627 AddRec->getStart(), AddRec->getStepRecurrence(*
this), MaxBECount,
4629 if (!RangeFromFactoring.isFullSet())
4630 ConservativeResult =
4635 return setRange(AddRec, SignHint, ConservativeResult);
4638 if (
const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
4642 ConservativeResult = ConservativeResult.intersectWith(MDRange.
getValue());
4648 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
4650 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
4652 if (Ones != ~Zeros + 1)
4653 ConservativeResult =
4654 ConservativeResult.intersectWith(
ConstantRange(Ones, ~Zeros + 1));
4656 assert(SignHint == ScalarEvolution::HINT_RANGE_SIGNED &&
4657 "generalize as needed!");
4660 ConservativeResult = ConservativeResult.intersectWith(
4665 return setRange(U, SignHint, ConservativeResult);
4668 return setRange(S, SignHint, ConservativeResult);
4673 const SCEV *MaxBECount,
4674 unsigned BitWidth) {
4675 assert(!isa<SCEVCouldNotCompute>(MaxBECount) &&
4694 StartURange.
add(MaxBECountRange.multiply(StepSRange));
4699 if (ZExtStartURange.
add(ZExtMaxBECountRange.
multiply(SExtStepSRange)) ==
4705 bool IsFullRange = Min.
isMinValue() && Max.isMaxValue();
4713 StartSRange.
add(MaxBECountRange.multiply(StepSRange));
4720 if (SExtStartSRange.
add(ZExtMaxBECountRange.
multiply(SExtStepSRange)) ==
4737 const SCEV *MaxBECount,
4738 unsigned BitWidth) {
4742 struct SelectPattern {
4743 Value *Condition =
nullptr;
4756 if (
auto *SA = dyn_cast<SCEVAddExpr>(S)) {
4759 if (SA->getNumOperands() != 2 || !isa<SCEVConstant>(SA->getOperand(0)))
4762 Offset = cast<SCEVConstant>(SA->getOperand(0))->getAPInt();
4763 S = SA->getOperand(1);
4767 if (
auto *SCast = dyn_cast<SCEVCastExpr>(S)) {
4768 CastOp = SCast->getSCEVType();
4769 S = SCast->getOperand();
4772 using namespace llvm::PatternMatch;
4775 const APInt *TrueVal, *FalseVal;
4779 Condition =
nullptr;
4783 TrueValue = *TrueVal;
4784 FalseValue = *FalseVal;
4793 TrueValue = TrueValue.trunc(BitWidth);
4794 FalseValue = FalseValue.trunc(BitWidth);
4797 TrueValue = TrueValue.zext(BitWidth);
4798 FalseValue = FalseValue.zext(BitWidth);
4801 TrueValue = TrueValue.sext(BitWidth);
4802 FalseValue = FalseValue.sext(BitWidth);
4811 bool isRecognized() {
return Condition !=
nullptr; }
4814 SelectPattern StartPattern(*
this, BitWidth, Start);
4815 if (!StartPattern.isRecognized())
4818 SelectPattern StepPattern(*
this, BitWidth, Step);
4819 if (!StepPattern.isRecognized())
4822 if (StartPattern.Condition != StepPattern.Condition) {
4843 this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount, BitWidth);
4845 this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount, BitWidth);
4847 return TrueRange.unionWith(FalseRange);
4866 bool ScalarEvolution::isSCEVExprNeverPoison(
const Instruction *I) {
4873 if (InnermostContainingLoop ==
nullptr ||
4895 for (
unsigned OpIndex = 0; OpIndex < I->
getNumOperands(); ++OpIndex) {
4901 if (
auto *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
4902 bool AllOtherOpsLoopInvariant =
true;
4903 for (
unsigned OtherOpIndex = 0; OtherOpIndex < I->
getNumOperands();
4905 if (OtherOpIndex != OpIndex) {
4908 AllOtherOpsLoopInvariant =
false;
4913 if (AllOtherOpsLoopInvariant &&
4921 bool ScalarEvolution::isAddRecNeverPoison(
const Instruction *I,
const Loop *L) {
4923 if (isSCEVExprNeverPoison(I))
4950 if (!ExitingBB || !LatchBB || ExitingBB != LatchBB)
4962 bool LatchControlDependentOnPoison =
false;
4963 while (!PoisonStack.
empty() && !LatchControlDependentOnPoison) {
4966 for (
auto *PoisonUser : Poison->
users()) {
4968 if (Pushed.
insert(cast<Instruction>(PoisonUser)).second)
4969 PoisonStack.
push_back(cast<Instruction>(PoisonUser));
4970 }
else if (
auto *BI = dyn_cast<BranchInst>(PoisonUser)) {
4971 assert(BI->isConditional() &&
"Only possibility!");
4972 if (BI->getParent() == LatchBB) {
4973 LatchControlDependentOnPoison =
true;
4980 return LatchControlDependentOnPoison && loopHasNoAbnormalExits(L);
4983 ScalarEvolution::LoopProperties
4984 ScalarEvolution::getLoopProperties(
const Loop *L) {
4985 typedef ScalarEvolution::LoopProperties LoopProperties;
4987 auto Itr = LoopPropertiesCache.find(L);
4988 if (Itr == LoopPropertiesCache.end()) {
4990 if (
auto *SI = dyn_cast<StoreInst>(I))
4991 return !
SI->isSimple();
4996 LoopProperties LP = {
true,
5000 for (
auto &I : *BB) {
5002 LP.HasNoAbnormalExits =
false;
5003 if (HasSideEffects(&I))
5004 LP.HasNoSideEffects =
false;
5005 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
5009 auto InsertPair = LoopPropertiesCache.insert({
L, LP});
5010 assert(InsertPair.second &&
"We just checked!");
5011 Itr = InsertPair.first;
5017 const SCEV *ScalarEvolution::createSCEV(
Value *V) {
5028 }
else if (
ConstantInt *CI = dyn_cast<ConstantInt>(V))
5030 else if (isa<ConstantPointerNull>(V))
5032 else if (
GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
5034 else if (!isa<ConstantExpr>(V))
5039 switch (BO->Opcode) {
5050 if (
auto *OpSCEV = getExistingSCEV(BO->Op)) {
5066 if (BO->Opcode == Instruction::Sub)
5074 if (BO->Opcode == Instruction::Sub)
5081 NewBO->Opcode != Instruction::Sub)) {
5091 case Instruction::Mul: {
5095 if (
auto *OpSCEV = getExistingSCEV(BO->Op)) {
5110 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
5119 case Instruction::UDiv:
5121 case Instruction::Sub: {
5124 Flags = getNoWrapFlagsFromUB(BO->Op);
5130 if (
ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
5131 if (CI->isNullValue())
5133 if (CI->isAllOnesValue())
5135 const APInt &
A = CI->getValue();
5144 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
5146 0, &AC,
nullptr, &DT);
5148 APInt EffectiveMask =
5150 if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) {
5171 if (
ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
5173 const APInt &CIVal = CI->getValue();
5191 if (
ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
5193 if (CI->isAllOnesValue())
5200 if (
auto *LBO = dyn_cast<BinaryOperator>(BO->LHS))
5201 if (
ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1)))
5203 LCI->getValue() == CI->getValue())
5205 dyn_cast<SCEVZeroExtendExpr>(
getSCEV(BO->LHS))) {
5206 Type *UTy = BO->LHS->getType();
5207 const SCEV *Z0 = Z->getOperand();
5220 APInt Trunc = CI->getValue().
trunc(Z0TySize);
5229 case Instruction::Shl:
5231 if (
ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) {
5238 if (SA->getValue().uge(BitWidth))
5248 if (BO->Op && SA->getValue().ult(BitWidth - 1))
5249 Flags = getNoWrapFlagsFromUB(BO->Op);
5257 case Instruction::AShr:
5259 if (
ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS))
5260 if (
Operator *L = dyn_cast<Operator>(BO->LHS))
5261 if (L->getOpcode() == Instruction::Shl &&
5262 L->getOperand(1) == BO->RHS) {
5269 if (CI->getValue().uge(BitWidth))
5272 uint64_t Amt = BitWidth - CI->getZExtValue();
5273 if (Amt == BitWidth)
5274 return getSCEV(L->getOperand(0));
5285 case Instruction::Trunc:
5288 case Instruction::ZExt:
5291 case Instruction::SExt:
5294 case Instruction::BitCast:
5305 case Instruction::GetElementPtr:
5306 return createNodeForGEP(cast<GEPOperator>(U));
5308 case Instruction::PHI:
5309 return createNodeForPHI(cast<PHINode>(U));
5316 if (isa<Instruction>(U))
5317 return createNodeForSelectOrPHI(cast<Instruction>(U), U->
getOperand(0),
5322 case Instruction::Invoke:
5361 assert(ExitingBlock &&
"Must pass a non-null exiting block!");
5363 "Exiting block must actually branch out of the loop!");
5370 const auto *MaxExitCount =
5398 assert(ExitingBlock &&
"Must pass a non-null exiting block!");
5400 "Exiting block must actually branch out of the loop!");
5409 if (
const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul))
5410 TCMul = Mul->getOperand(0);
5432 return getBackedgeTakenInfo(L).getExact(ExitingBlock,
this);
5438 return getPredicatedBackedgeTakenInfo(L).getExact(
this, &Preds);
5442 return getBackedgeTakenInfo(L).getExact(
this);
5448 return getBackedgeTakenInfo(L).getMax(
this);
5452 return getBackedgeTakenInfo(L).isMaxOrZero(
this);
5466 const ScalarEvolution::BackedgeTakenInfo &
5467 ScalarEvolution::getPredicatedBackedgeTakenInfo(
const Loop *L) {
5468 auto &BTI = getBackedgeTakenInfo(L);
5469 if (BTI.hasFullInfo())
5472 auto Pair = PredicatedBackedgeTakenCounts.insert({
L, BackedgeTakenInfo()});
5475 return Pair.first->second;
5477 BackedgeTakenInfo Result =
5478 computeBackedgeTakenCount(L,
true);
5480 return PredicatedBackedgeTakenCounts.find(L)->second = std::move(Result);
5483 const ScalarEvolution::BackedgeTakenInfo &
5484 ScalarEvolution::getBackedgeTakenInfo(
const Loop *L) {
5490 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator,
bool> Pair =
5491 BackedgeTakenCounts.insert({
L, BackedgeTakenInfo()});
5493 return Pair.first->second;
5498 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
5503 "Computed backedge-taken count isn't loop invariant for loop!");
5504 ++NumTripCountsComputed;
5507 isa<PHINode>(L->
getHeader()->begin())) {
5509 ++NumTripCountsNotComputed;
5517 if (Result.hasAnyInfo()) {
5522 while (!Worklist.
empty()) {
5524 if (!Visited.
insert(I).second)
5528 ValueExprMap.
find_as(static_cast<Value *>(I));
5529 if (It != ValueExprMap.
end()) {
5530 const SCEV *Old = It->second;
5538 if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) {
5540 forgetMemoizedResults(Old);
5542 if (
PHINode *PN = dyn_cast<PHINode>(I))
5543 ConstantEvolutionLoopExitValue.erase(PN);
5555 return BackedgeTakenCounts.find(L)->second = std::move(Result);
5560 auto RemoveLoopFromBackedgeMap =
5562 auto BTCPos = Map.find(L);
5563 if (BTCPos != Map.end()) {
5564 BTCPos->second.clear();
5569 RemoveLoopFromBackedgeMap(BackedgeTakenCounts);
5570 RemoveLoopFromBackedgeMap(PredicatedBackedgeTakenCounts);
5577 while (!Worklist.
empty()) {
5579 if (!Visited.
insert(I).second)
5583 ValueExprMap.
find_as(static_cast<Value *>(I));
5584 if (It != ValueExprMap.
end()) {
5586 forgetMemoizedResults(It->second);
5587 if (
PHINode *PN = dyn_cast<PHINode>(I))
5588 ConstantEvolutionLoopExitValue.erase(PN);
5599 LoopPropertiesCache.erase(L);
5611 while (!Worklist.
empty()) {
5613 if (!Visited.
insert(I).second)
5617 ValueExprMap.
find_as(static_cast<Value *>(I));
5618 if (It != ValueExprMap.
end()) {
5620 forgetMemoizedResults(It->second);
5621 if (
PHINode *PN = dyn_cast<PHINode>(I))
5622 ConstantEvolutionLoopExitValue.erase(PN);
5641 if (!isComplete() || ExitNotTaken.empty())
5644 const SCEV *BECount =
nullptr;
5645 for (
auto &ENT : ExitNotTaken) {
5649 BECount = ENT.ExactNotTaken;
5650 else if (BECount != ENT.ExactNotTaken)
5652 if (Preds && !ENT.hasAlwaysTruePredicate())
5653 Preds->
add(ENT.Predicate.get());
5655 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
5656 "Predicate should be always true!");
5659 assert(BECount &&
"Invalid not taken count for loop exit");
5665 ScalarEvolution::BackedgeTakenInfo::getExact(
BasicBlock *ExitingBlock,
5667 for (
auto &ENT : ExitNotTaken)
5668 if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePredicate())
5669 return ENT.ExactNotTaken;
5676 ScalarEvolution::BackedgeTakenInfo::getMax(
ScalarEvolution *SE)
const {
5677 auto PredicateNotAlwaysTrue = [](
const ExitNotTakenInfo &ENT) {
5678 return !ENT.hasAlwaysTruePredicate();
5681 if (
any_of(ExitNotTaken, PredicateNotAlwaysTrue) || !getMax())
5687 bool ScalarEvolution::BackedgeTakenInfo::isMaxOrZero(
ScalarEvolution *SE)
const {
5688 auto PredicateNotAlwaysTrue = [](
const ExitNotTakenInfo &ENT) {
5689 return !ENT.hasAlwaysTruePredicate();
5691 return MaxOrZero && !
any_of(ExitNotTaken, PredicateNotAlwaysTrue);
5694 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(
const SCEV *S,
5700 for (
auto &ENT : ExitNotTaken)
5710 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
5713 bool Complete,
const SCEV *MaxCount,
bool MaxOrZero)
5714 : MaxAndComplete(MaxCount, Complete), MaxOrZero(MaxOrZero) {
5715 typedef ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo EdgeExitInfo;
5716 ExitNotTaken.reserve(ExitCounts.size());
5718 ExitCounts.begin(), ExitCounts.end(), std::back_inserter(ExitNotTaken),
5719 [&](
const EdgeExitInfo &EEI) {
5721 const ExitLimit &EL = EEI.second;
5722 if (EL.Predicates.empty())
5723 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
nullptr);
5726 for (
auto *Pred : EL.Predicates)
5729 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken, std::move(
Predicate));
5735 ExitNotTaken.clear();
5739 ScalarEvolution::BackedgeTakenInfo
5740 ScalarEvolution::computeBackedgeTakenCount(
const Loop *L,
5741 bool AllowPredicates) {
5745 typedef ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo EdgeExitInfo;
5748 bool CouldComputeBECount =
true;
5750 const SCEV *MustExitMaxBECount =
nullptr;
5751 const SCEV *MayExitMaxBECount =
nullptr;
5752 bool MustExitMaxOrZero =
false;
5757 for (
unsigned i = 0, e = ExitingBlocks.
size();
i != e; ++
i) {
5759 ExitLimit EL = computeExitLimit(L, ExitBB, AllowPredicates);
5761 assert((AllowPredicates || EL.Predicates.empty()) &&
5762 "Predicated exit limit when predicates are not allowed!");
5769 CouldComputeBECount =
false;
5785 if (!MustExitMaxBECount) {
5786 MustExitMaxBECount = EL.MaxNotTaken;
5787 MustExitMaxOrZero = EL.MaxOrZero;
5789 MustExitMaxBECount =
5794 MayExitMaxBECount = EL.MaxNotTaken;
5801 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
5805 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.
size() == 1);
5806 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
5807 MaxBECount, MaxOrZero);
5810 ScalarEvolution::ExitLimit
5811 ScalarEvolution::computeExitLimit(
const Loop *L,
BasicBlock *ExitingBlock,
5812 bool AllowPredicates) {
5817 bool MustExecuteLoopHeader =
true;
5825 MustExecuteLoopHeader =
false;
5845 if (!MustExecuteLoopHeader && ExitingBlock != L->
getHeader()) {
5874 if (
BranchInst *BI = dyn_cast<BranchInst>(Term)) {
5875 assert(BI->isConditional() &&
"If unconditional, it can't be in loop!");
5877 return computeExitLimitFromCond(
5878 L, BI->getCondition(), BI->getSuccessor(0), BI->getSuccessor(1),
5879 IsOnlyExit, AllowPredicates);
5882 if (
SwitchInst *SI = dyn_cast<SwitchInst>(Term))
5883 return computeExitLimitFromSingleExitSwitch(L, SI, Exit,
5889 ScalarEvolution::ExitLimit
5890 ScalarEvolution::computeExitLimitFromCond(
const Loop *L,
5895 bool AllowPredicates) {
5900 bool EitherMayExit = L->
contains(TBB);
5901 ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
5902 ControlsExit && !EitherMayExit,
5904 ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
5905 ControlsExit && !EitherMayExit,
5909 if (EitherMayExit) {
5919 MaxBECount = EL1.MaxNotTaken;
5921 MaxBECount = EL0.MaxNotTaken;
5928 assert(L->
contains(FBB) &&
"Loop block has no successor in loop!");
5929 if (EL0.MaxNotTaken == EL1.MaxNotTaken)
5930 MaxBECount = EL0.MaxNotTaken;
5931 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
5932 BECount = EL0.ExactNotTaken;
5940 if (isa<SCEVCouldNotCompute>(MaxBECount) &&
5941 !isa<SCEVCouldNotCompute>(BECount))
5942 MaxBECount = BECount;
5944 return ExitLimit(BECount, MaxBECount,
false,
5945 {&EL0.Predicates, &EL1.Predicates});
5949 bool EitherMayExit = L->
contains(FBB);
5950 ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
5951 ControlsExit && !EitherMayExit,
5953 ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
5954 ControlsExit && !EitherMayExit,
5958 if (EitherMayExit) {
5968 MaxBECount = EL1.MaxNotTaken;
5970 MaxBECount = EL0.MaxNotTaken;
5977 assert(L->
contains(TBB) &&
"Loop block has no successor in loop!");
5978 if (EL0.MaxNotTaken == EL1.MaxNotTaken)
5979 MaxBECount = EL0.MaxNotTaken;
5980 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
5981 BECount = EL0.ExactNotTaken;
5984 return ExitLimit(BECount, MaxBECount,
false,
5985 {&EL0.Predicates, &EL1.Predicates});
5991 if (
ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond)) {
5993 computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit);
5994 if (EL.hasFullInfo() || !AllowPredicates)
5998 return computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit,
6006 if (
ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
6007 if (L->
contains(FBB) == !CI->getZExtValue())
6012 return getZero(CI->getType());
6016 return computeExitCountExhaustively(L, ExitCond, !L->
contains(TBB));
6019 ScalarEvolution::ExitLimit
6020 ScalarEvolution::computeExitLimitFromICmp(
const Loop *L,
6025 bool AllowPredicates) {
6038 computeLoadConstantCompareExitLimit(LI, RHS, L, Cond);
6039 if (ItCnt.hasAnyInfo())
6063 if (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
6064 if (
const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
6065 if (AddRec->getLoop() ==
L) {
6070 const SCEV *
Ret = AddRec->getNumIterationsInRange(CompRange, *
this);
6071 if (!isa<SCEVCouldNotCompute>(Ret))
return Ret;
6077 ExitLimit EL = howFarToZero(
getMinusSCEV(LHS, RHS), L, ControlsExit,
6079 if (EL.hasAnyInfo())
return EL;
6084 ExitLimit EL = howFarToNonZero(
getMinusSCEV(LHS, RHS), L);
6085 if (EL.hasAnyInfo())
return EL;
6091 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, ControlsExit,
6093 if (EL.hasAnyInfo())
return EL;
6100 howManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit,
6102 if (EL.hasAnyInfo())
return EL;
6109 auto *ExhaustiveCount =
6110 computeExitCountExhaustively(L, ExitCond, !L->
contains(TBB));
6112 if (!isa<SCEVCouldNotCompute>(ExhaustiveCount))
6113 return ExhaustiveCount;
6115 return computeShiftCompareExitLimit(ExitCond->
getOperand(0),
6119 ScalarEvolution::ExitLimit
6120 ScalarEvolution::computeExitLimitFromSingleExitSwitch(
const Loop *L,
6123 bool ControlsExit) {
6131 "Default case must not exit the loop!");
6136 ExitLimit EL = howFarToZero(
getMinusSCEV(LHS, RHS), L, ControlsExit);
6137 if (EL.hasAnyInfo())
6148 assert(isa<SCEVConstant>(Val) &&
6149 "Evaluation of SCEV at constant didn't fold correctly?");
6150 return cast<SCEVConstant>(Val)->getValue();
6155 ScalarEvolution::ExitLimit
6156 ScalarEvolution::computeLoadConstantCompareExitLimit(
6174 !cast<Constant>(GEP->
getOperand(1))->isNullValue())
6178 Value *VarIdx =
nullptr;
6179 std::vector<Constant*> Indexes;
6180 unsigned VarIdxNum = 0;
6183 Indexes.push_back(CI);
6184 }
else if (!isa<ConstantInt>(GEP->
getOperand(
i))) {
6188 Indexes.push_back(
nullptr);
6204 !isa<SCEVConstant>(IdxExpr->
getOperand(0)) ||
6209 for (
unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) {
6211 cast<IntegerType>(IdxExpr->
getType()), IterationNum);
6215 Indexes[VarIdxNum] = Val;
6223 if (!isa<ConstantInt>(Result))
break;
6224 if (cast<ConstantInt>(Result)->getValue().isMinValue()) {
6225 ++NumArrayLenItCounts;
6232 ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
6248 auto MatchPositiveShift =
6251 using namespace PatternMatch;
6255 OutOpCode = Instruction::LShr;
6257 OutOpCode = Instruction::AShr;
6259 OutOpCode = Instruction::Shl;
6274 auto MatchShiftRecurrence =
6291 if (MatchPositiveShift(LHS, V, OpC)) {
6292 PostShiftOpCode = OpC;
6298 if (!PNOut || PNOut->getParent() != L->
getHeader())
6301 Value *BEValue = PNOut->getIncomingValueForBlock(Latch);
6307 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) &&
6314 (!PostShiftOpCode.
hasValue() || *PostShiftOpCode == OpCodeOut);
6319 if (!MatchShiftRecurrence(LHS, PN, OpCode))
6336 case Instruction::AShr: {
6340 bool KnownZero, KnownOne;
6343 auto *Ty = cast<IntegerType>(RHS->
getType());
6353 case Instruction::LShr:
6354 case Instruction::Shl:
6364 "Otherwise cannot be an operand to a branch instruction");
6368 const SCEV *UpperBound =
6379 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) ||
6380 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
6384 if (
const CallInst *CI = dyn_cast<CallInst>(I))
6385 if (
const Function *F = CI->getCalledFunction())
6396 if (isa<PHINode>(I)) {
6417 if (isa<Constant>(Op))
continue;
6427 P = PHIMap.
lookup(OpInst);
6436 if (PHI && PHI != P)
6453 if (
PHINode *PN = dyn_cast<PHINode>(I))
6470 if (
Constant *
C = dyn_cast<Constant>(V))
return C;
6472 if (!I)
return nullptr;
6483 if (isa<PHINode>(I))
return nullptr;
6491 if (!Operands[
i])
return nullptr;
6496 if (!C)
return nullptr;
6500 if (
CmpInst *CI = dyn_cast<CmpInst>(I))
6502 Operands[1], DL, TLI);
6503 if (
LoadInst *LI = dyn_cast<LoadInst>(I)) {
6524 if (IncomingVal != CurrentVal) {
6527 IncomingVal = CurrentVal;
6539 ScalarEvolution::getConstantEvolutionLoopExitValue(
PHINode *PN,
6542 auto I = ConstantEvolutionLoopExitValue.find(PN);
6543 if (I != ConstantEvolutionLoopExitValue.end())
6547 return ConstantEvolutionLoopExitValue[PN] =
nullptr;
6549 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
6553 assert(PN->
getParent() == Header &&
"Can't evaluate PHI not in loop header!");
6559 for (
auto &I : *Header) {
6563 if (!StartCST)
continue;
6564 CurrentIterVals[PHI] = StartCST;
6566 if (!CurrentIterVals.
count(PN))
6567 return RetVal =
nullptr;
6573 return RetVal =
nullptr;
6576 unsigned IterationNum = 0;
6578 for (; ; ++IterationNum) {
6579 if (IterationNum == NumIterations)
6580 return RetVal = CurrentIterVals[PN];
6589 NextIterVals[PN] = NextPHI;
6591 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
6597 for (
const auto &I : CurrentIterVals) {
6599 if (!PHI || PHI == PN || PHI->
getParent() != Header)
continue;
6604 for (
const auto &I : PHIsToCompute) {
6606 Constant *&NextPHI = NextIterVals[PHI];
6611 if (NextPHI != I.second)
6612 StoppedEvolving =
false;
6617 if (StoppedEvolving)
6618 return RetVal = CurrentIterVals[PN];
6620 CurrentIterVals.swap(NextIterVals);
6624 const SCEV *ScalarEvolution::computeExitCountExhaustively(
const Loop *L,
6636 assert(PN->
getParent() == Header &&
"Can't evaluate PHI not in loop header!");
6639 assert(Latch &&
"Should follow from NumIncomingValues == 2!");
6641 for (
auto &I : *Header) {
6646 if (!StartCST)
continue;
6647 CurrentIterVals[PHI] = StartCST;
6649 if (!CurrentIterVals.
count(PN))
6657 for (
unsigned IterationNum = 0; IterationNum !=
MaxIterations;++IterationNum){
6658 auto *CondVal = dyn_cast_or_null<ConstantInt>(
6664 if (CondVal->getValue() == uint64_t(ExitWhen)) {
6665 ++NumBruteForceTripCountsComputed;
6676 for (
const auto &I : CurrentIterVals) {
6678 if (!PHI || PHI->
getParent() != Header)
continue;
6681 for (
PHINode *PHI : PHIsToCompute) {
6682 Constant *&NextPHI = NextIterVals[PHI];
6683 if (NextPHI)
continue;
6688 CurrentIterVals.swap(NextIterVals);
6699 for (
auto &LS : Values)
6701 return LS.second ? LS.second : V;
6703 Values.emplace_back(L,
nullptr);
6706 const SCEV *
C = computeSCEVAtScope(V, L);
6707 for (
auto &LS :
reverse(ValuesAtScopes[V]))
6708 if (LS.first == L) {
6720 switch (static_cast<SCEVTypes>(V->
getSCEVType())) {
6725 return cast<SCEVConstant>(V)->getValue();
6749 if (
PointerType *PTy = dyn_cast<PointerType>(
C->getType())) {
6750 unsigned AS = PTy->getAddressSpace();
6756 if (!C2)
return nullptr;
6773 if (
PointerType *PTy = dyn_cast<PointerType>(
C->getType())) {
6774 if (PTy->getElementType()->isStructTy())
6789 if (
C->getType()->isPointerTy())
return nullptr;
6814 const SCEV *ScalarEvolution::computeSCEVAtScope(
const SCEV *V,
const Loop *L) {
6815 if (isa<SCEVConstant>(V))
return V;
6819 if (
const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
6820 if (
Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
6823 if (
PHINode *PN = dyn_cast<PHINode>(I))
6831 dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
6836 getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI);
6847 bool MadeImprovement =
false;
6849 if (
Constant *
C = dyn_cast<Constant>(Op)) {
6862 MadeImprovement |= OrigV != OpV;
6875 if (MadeImprovement) {
6878 if (
const CmpInst *CI = dyn_cast<CmpInst>(I))
6880 Operands[1], DL, &TLI);
6881 else if (
const LoadInst *LI = dyn_cast<LoadInst>(I)) {
6882 if (!LI->isVolatile())
6899 for (
unsigned i = 0, e = Comm->getNumOperands();
i != e; ++
i) {
6901 if (OpAtScope != Comm->getOperand(
i)) {
6905 Comm->op_begin()+
i);
6908 for (++
i;
i != e; ++
i) {
6910 NewOps.push_back(OpAtScope);
6912 if (isa<SCEVAddExpr>(Comm))
6914 if (isa<SCEVMulExpr>(Comm))
6916 if (isa<SCEVSMaxExpr>(Comm))
6918 if (isa<SCEVUMaxExpr>(Comm))
6927 if (
const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) {
6930 if (LHS == Div->getLHS() && RHS == Div->getRHS())
6937 if (
const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
6941 for (
unsigned i = 0, e = AddRec->getNumOperands();
i != e; ++
i) {
6943 if (OpAtScope == AddRec->getOperand(
i))
6949 AddRec->op_begin()+
i);
6951 for (++
i;
i != e; ++
i)
6954 const SCEV *FoldedRec =
6968 if (!AddRec->getLoop()->contains(L)) {
6975 return AddRec->evaluateAtIteration(BackedgeTakenCount, *
this);
6983 if (Op == Cast->getOperand())
6990 if (Op == Cast->getOperand())
6997 if (Op == Cast->getOperand())
7021 assert(A != 0 &&
"A must be non-zero.");
7044 APInt Mod(BW + 1, 0);
7069 if (!LC || !MC || !NC)
7072 uint32_t BitWidth = LC->getAPInt().getBitWidth();
7073 const APInt &L = LC->getAPInt();
7074 const APInt &M = MC->getAPInt();
7075 const APInt &
N = NC->getAPInt();
7076 APInt Two(BitWidth, 2);
7077 APInt Four(BitWidth, 4);
7080 using namespace APIntOps;
7093 SqrtTerm -= Four * (A *
C);
7095 if (SqrtTerm.isNegative()) {
7102 APInt SqrtVal(SqrtTerm.sqrt());
7118 return std::make_pair(cast<SCEVConstant>(SE.
getConstant(Solution1)),
7123 ScalarEvolution::ExitLimit
7124 ScalarEvolution::howFarToZero(
const SCEV *V,
const Loop *L,
bool ControlsExit,
7125 bool AllowPredicates) {
7134 if (
const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
7136 if (C->getValue()->isZero())
return C;
7141 if (!AddRec && AllowPredicates)
7147 if (!AddRec || AddRec->
getLoop() !=
L)
7159 if (!CB->getZExtValue())
7168 return ExitLimit(R1, R1,
false, Predicates);
7200 if (!StepC || StepC->getValue()->equalsInt(0))
7208 bool CountDown = StepC->getAPInt().isNegative();
7214 if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) {
7233 return ExitLimit(Distance,
getConstant(MaxBECount),
false, Predicates);
7241 const APInt &StepV = StepC->getAPInt();
7284 auto *WideTy = Distance->
getType();
7288 return ExitLimit(Limit, Limit,
false, Predicates);
7298 loopHasNoAbnormalExits(AddRec->
getLoop())) {
7301 return ExitLimit(Exact, Exact,
false, Predicates);
7305 if (
const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start)) {
7308 return ExitLimit(E, E,
false, Predicates);
7313 ScalarEvolution::ExitLimit
7314 ScalarEvolution::howFarToNonZero(
const SCEV *V,
const Loop *L) {
7321 if (
const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
7332 std::pair<BasicBlock *, BasicBlock *>
7333 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(
BasicBlock *BB) {
7343 if (
Loop *L = LI.getLoopFor(BB))
7346 return {
nullptr,
nullptr};
7356 if (A == B)
return true;
7362 return A->
isIdenticalTo(B) && (isa<BinaryOperator>(
A) || isa<GetElementPtrInst>(A));
7367 if (
const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
7368 if (
const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
7369 if (
const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
7370 if (
const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
7371 if (ComputesEqualValues(AI, BI))
7379 const SCEV *&LHS,
const SCEV *&RHS,
7381 bool Changed =
false;
7388 if (
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
7390 if (
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
7393 RHSC->getValue())->isNullValue())
7394 goto trivially_false;
7396 goto trivially_true;
7418 if (
const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
7419 const APInt &RA = RC->getAPInt();
7421 bool SimplifiedByConstantRange =
false;
7426 goto trivially_true;
7428 goto trivially_false;
7437 Changed = SimplifiedByConstantRange =
true;
7441 if (!SimplifiedByConstantRange) {
7449 if (
const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS))
7451 dyn_cast<SCEVMulExpr>(AE->getOperand(0)))
7452 if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 &&
7453 ME->getOperand(0)->isAllOnesValue()) {
7454 RHS = AE->getOperand(1);
7455 LHS = ME->getOperand(1);
7497 goto trivially_true;
7499 goto trivially_false;
7511 }
else if (!
getSignedRange(LHS).getSignedMin().isMinSignedValue()) {
7602 const SCEV *LHS,
const SCEV *RHS) {
7612 bool LeftGuarded =
false;
7613 bool RightGuarded =
false;
7618 if (!RAR)
return true;
7623 const Loop *L = RAR->getLoop();
7626 if (!LAR)
return true;
7627 RightGuarded =
true;
7630 if (LeftGuarded && RightGuarded)
7633 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
7637 return isKnownPredicateViaConstantRanges(Pred, LHS, RHS);
7640 bool ScalarEvolution::isMonotonicPredicate(
const SCEVAddRecExpr *LHS,
7643 bool Result = isMonotonicPredicateImpl(LHS, Pred, Increasing);
7648 bool IncreasingSwapped;
7649 bool ResultSwapped = isMonotonicPredicateImpl(
7652 assert(Result == ResultSwapped &&
"should be able to analyze both!");
7654 assert(Increasing == !IncreasingSwapped &&
7655 "monotonicity should flip as we flip the predicate");
7661 bool ScalarEvolution::isMonotonicPredicateImpl(
const SCEVAddRecExpr *LHS,
7719 const SCEV *&InvariantRHS) {
7731 if (!ArLHS || ArLHS->
getLoop() !=
L)
7735 if (!isMonotonicPredicate(ArLHS, Pred, Increasing))
7761 InvariantPred = Pred;
7767 bool ScalarEvolution::isKnownPredicateViaConstantRanges(
7803 auto MatchBinaryAddToConst =
7806 const SCEV *NonConstOp, *ConstOp;
7809 if (!splitBinaryAdd(Result, ConstOp, NonConstOp, FlagsPresent) ||
7810 !isa<SCEVConstant>(ConstOp) || NonConstOp != X)
7813 OutY = cast<SCEVConstant>(ConstOp)->getAPInt();
7814 return (FlagsPresent & ExpectedFlags) == ExpectedFlags;
7875 bool ScalarEvolution::isImpliedViaGuard(
BasicBlock *BB,
7877 const SCEV *LHS,
const SCEV *RHS) {
7883 using namespace llvm::PatternMatch;
7886 return match(&I, m_Intrinsic<Intrinsic::experimental_guard>(
7888 isImpliedCond(Pred, LHS, RHS, Condition,
false);
7898 const SCEV *LHS,
const SCEV *RHS) {
7901 if (!L)
return true;
7903 if (isKnownPredicateViaConstantRanges(Pred, LHS, RHS))
7912 if (LoopContinuePredicate && LoopContinuePredicate->
isConditional() &&
7913 isImpliedCond(Pred, LHS, RHS,
7920 if (WalkingBEDominatingConds)
7926 const auto &BETakenInfo = getBackedgeTakenInfo(L);
7927 const SCEV *LatchBECount = BETakenInfo.getExact(Latch,
this);
7934 const SCEV *LoopCounter =
7942 for (
auto &AssumeVH : AC.assumptions()) {
7945 auto *CI = cast<CallInst>(AssumeVH);
7949 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0),
false))
7959 if (isImpliedViaGuard(Latch, Pred, LHS, RHS))
7963 DTN != HeaderDTN; DTN = DTN->
getIDom()) {
7965 assert(DTN &&
"should reach the loop header before reaching the root!");
7968 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
7976 if (!ContinuePredicate || !ContinuePredicate->
isConditional())
7992 if (isImpliedCond(Pred, LHS, RHS, Condition,
8004 const SCEV *LHS,
const SCEV *RHS) {
8007 if (!L)
return false;
8009 if (isKnownPredicateViaConstantRanges(Pred, LHS, RHS))
8015 for (std::pair<BasicBlock *, BasicBlock *>
8018 Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
8020 if (isImpliedViaGuard(Pair.first, Pred, LHS, RHS))
8025 if (!LoopEntryPredicate ||
8029 if (isImpliedCond(Pred, LHS, RHS,
8036 for (
auto &AssumeVH : AC.assumptions()) {
8039 auto *CI = cast<CallInst>(AssumeVH);
8043 if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0),
false))
8052 Value *FoundCondValue,
8054 if (!PendingLoopPredicates.insert(FoundCondValue).second)
8058 make_scope_exit([&]() { PendingLoopPredicates.erase(FoundCondValue); });
8061 if (
BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) {
8064 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
8065 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
8068 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
8069 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
8074 if (!ICI)
return false;
8087 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS);
8093 const SCEV *FoundLHS,
8094 const SCEV *FoundRHS) {
8122 if (FoundLHS == FoundRHS)
8126 if (LHS == FoundRHS || RHS == FoundLHS) {
8127 if (isa<SCEVConstant>(RHS)) {
8137 if (FoundPred == Pred)
8138 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS);
8143 if (isa<SCEVConstant>(RHS))
8144 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS);
8147 RHS, LHS, FoundLHS, FoundRHS);
8155 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS);
8159 (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) {
8162 const SCEV *V =
nullptr;
8164 if (isa<SCEVConstant>(FoundLHS)) {
8165 C = cast<SCEVConstant>(FoundLHS);
8168 C = cast<SCEVConstant>(FoundRHS);
8185 APInt SharperMin = Min + 1;
8192 if (isImpliedCondOperands(Pred, LHS, RHS, V,
8207 if (isImpliedCondOperands(Pred, LHS, RHS, V,
getConstant(Min)))
8220 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS))
8224 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS))
8231 bool ScalarEvolution::splitBinaryAdd(
const SCEV *Expr,
8235 if (!AE || AE->getNumOperands() != 2)
8238 L = AE->getOperand(0);
8239 R = AE->getOperand(1);
8240 Flags = AE->getNoWrapFlags();
8249 if (isa<SCEVAddRecExpr>(Less) && isa<SCEVAddRecExpr>(More)) {
8250 const auto *LAR = cast<SCEVAddRecExpr>(
Less);
8251 const auto *MAR = cast<SCEVAddRecExpr>(More);
8253 if (LAR->getLoop() != MAR->getLoop())
8258 if (!LAR->isAffine() || !MAR->isAffine())
8261 if (LAR->getStepRecurrence(*
this) != MAR->getStepRecurrence(*
this))
8264 Less = LAR->getStart();
8265 More = MAR->getStart();
8270 if (isa<SCEVConstant>(Less) && isa<SCEVConstant>(More)) {
8271 const auto &M = cast<SCEVConstant>(More)->getAPInt();
8272 const auto &L = cast<SCEVConstant>(
Less)->getAPInt();
8278 if (splitBinaryAdd(Less, L, R, Flags))
8279 if (
const auto *LC = dyn_cast<SCEVConstant>(L))
8281 return -(LC->getAPInt());
8283 if (splitBinaryAdd(More, L, R, Flags))
8284 if (
const auto *LC = dyn_cast<SCEVConstant>(L))
8286 return LC->getAPInt();
8291 bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(
8293 const SCEV *FoundLHS,
const SCEV *FoundRHS) {
8302 if (!AddRecFoundLHS)
8309 const Loop *L = AddRecFoundLHS->getLoop();
8310 if (L != AddRecLHS->getLoop())
8347 if (!LDiff || !RDiff || *LDiff != *RDiff)
8353 APInt FoundRHSLimit;
8356 FoundRHSLimit = -(*RDiff);
8369 const SCEV *FoundLHS,
8370 const SCEV *FoundRHS) {
8371 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS))
8374 if (isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS, FoundRHS))
8377 return isImpliedCondOperandsHelper(Pred, LHS, RHS,
8378 FoundLHS, FoundRHS) ||
8380 isImpliedCondOperandsHelper(Pred, LHS, RHS,
8403 template<
typename MaxExprType>
8405 const SCEV *Candidate) {
8406 const MaxExprType *MaxExpr =
dyn_cast<MaxExprType>(MaybeMaxExpr);
8407 if (!MaxExpr)
return false;
8409 return find(MaxExpr->operands(), Candidate) != MaxExpr->op_end();
8414 template<
typename MaxExprType>
8416 const SCEV *MaybeMinExpr,
8417 const SCEV *Candidate) {
8422 return IsMaxConsistingOf<MaxExprType>(MaybeMaxExpr, SE.
getNotSCEV(Candidate));
8427 const SCEV *LHS,
const SCEV *RHS) {
8462 const SCEV *LHS,
const SCEV *RHS) {
8473 IsMinConsistingOf<SCEVSMaxExpr>(SE, LHS, RHS) ||
8475 IsMaxConsistingOf<SCEVSMaxExpr>(RHS, LHS);
8483 IsMinConsistingOf<SCEVUMaxExpr>(SE, LHS, RHS) ||
8485 IsMaxConsistingOf<SCEVUMaxExpr>(RHS, LHS);
8494 const SCEV *FoundLHS,
8495 const SCEV *FoundRHS) {
8496 auto IsKnownPredicateFull =
8498 return isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
8501 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
8543 const SCEV *FoundLHS,
8544 const SCEV *FoundRHS) {
8545 if (!isa<SCEVConstant>(RHS) || !isa<SCEVConstant>(FoundRHS))
8554 APInt ConstFoundRHS = cast<SCEVConstant>(FoundRHS)->getAPInt();
8566 APInt ConstRHS = cast<SCEVConstant>(RHS)->getAPInt();
8572 return SatisfyingLHSRange.
contains(LHSRange);
8575 bool ScalarEvolution::doesIVOverflowOnLT(
const SCEV *RHS,
const SCEV *Stride,
8576 bool IsSigned,
bool NoWrap) {
8579 if (NoWrap)
return false;
8591 return (MaxValue - MaxStrideMinusOne).
slt(MaxRHS);
8600 return (MaxValue - MaxStrideMinusOne).
ult(MaxRHS);
8603 bool ScalarEvolution::doesIVOverflowOnGT(
const SCEV *RHS,
const SCEV *Stride,
8604 bool IsSigned,
bool NoWrap) {
8605 if (NoWrap)
return false;
8617 return (MinValue + MaxStrideMinusOne).
sgt(MinRHS);
8626 return (MinValue + MaxStrideMinusOne).
ugt(MinRHS);
8629 const SCEV *ScalarEvolution::computeBECount(
const SCEV *Delta,
const SCEV *Step,
8637 ScalarEvolution::ExitLimit
8638 ScalarEvolution::howManyLessThans(
const SCEV *LHS,
const SCEV *RHS,
8639 const Loop *L,
bool IsSigned,
8640 bool ControlsExit,
bool AllowPredicates) {
8647 bool PredicatedIV =
false;
8649 if (!IV && AllowPredicates) {
8654 PredicatedIV =
true;
8661 bool NoWrap = ControlsExit &&
8669 if (!PositiveStride) {
8713 !loopHasNoSideEffects(L))
8716 }
else if (!Stride->
isOne() &&
8717 doesIVOverflowOnLT(RHS, Stride, IsSigned, NoWrap))
8732 const SCEV *BECountIfBackedgeTaken =
8733 computeBECount(
getMinusSCEV(End, Start), Stride,
false);
8741 const SCEV *BECount;
8743 BECount = BECountIfBackedgeTaken;
8746 BECount = computeBECount(
getMinusSCEV(End, Start), Stride,
false);
8749 const SCEV *MaxBECount;
8750 bool MaxOrZero =
false;
8751 if (isa<SCEVConstant>(BECount))
8752 MaxBECount = BECount;
8753 else if (isa<SCEVConstant>(BECountIfBackedgeTaken)) {
8757 MaxBECount = BECountIfBackedgeTaken;
8767 APInt StrideForMaxBECount;
8770 StrideForMaxBECount =
8776 StrideForMaxBECount =
APInt(BitWidth, 1, IsSigned);
8789 MaxBECount = computeBECount(
getConstant(MaxEnd - MinStart),
8793 if (isa<SCEVCouldNotCompute>(MaxBECount))
8794 MaxBECount = BECount;
8796 return ExitLimit(BECount, MaxBECount, MaxOrZero, Predicates);
8799 ScalarEvolution::ExitLimit
8800 ScalarEvolution::howManyGreaterThans(
const SCEV *LHS,
const SCEV *RHS,
8801 const Loop *L,
bool IsSigned,
8802 bool ControlsExit,
bool AllowPredicates) {
8809 if (!IV && AllowPredicates)
8819 bool NoWrap = ControlsExit &&
8832 if (!Stride->
isOne() && doesIVOverflowOnGT(RHS, Stride, IsSigned, NoWrap))
8839 const SCEV *End = RHS;
8843 const SCEV *BECount = computeBECount(
getMinusSCEV(Start, End), Stride,
false);
8864 if (isa<SCEVConstant>(BECount))
8865 MaxBECount = BECount;
8867 MaxBECount = computeBECount(
getConstant(MaxStart - MinEnd),
8870 if (isa<SCEVCouldNotCompute>(MaxBECount))
8871 MaxBECount = BECount;
8873 return ExitLimit(BECount, MaxBECount,
false, Predicates);
8883 if (!
SC->getValue()->isZero()) {
8885 Operands[0] = SE.
getZero(
SC->getType());
8887 getNoWrapFlags(FlagNW));
8888 if (
const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Shifted))
8889 return ShiftedAddRec->getNumIterationsInRange(
8897 if (
any_of(operands(), [](
const SCEV *Op) {
return !isa<SCEVConstant>(
Op); }))
8917 APInt One(BitWidth,1);
8918 APInt A = cast<SCEVConstant>(getOperand(1))->getAPInt();
8929 if (Range.
contains(Val->getValue()))
8936 "Linear scev computation is off in a bad way!");
8938 }
else if (isQuadratic()) {
8955 if (!CB->getZExtValue())
8969 if (!Range.
contains(R1Val->getValue()))
8979 if (Range.
contains(R1Val->getValue()))
8992 if (
const auto *SU = dyn_cast<SCEVUnknown>(S))
8993 return isa<UndefValue>(SU->getValue());
8994 else if (
const auto *
SC = dyn_cast<SCEVConstant>(S))
8995 return isa<UndefValue>(
SC->getValue());
9002 struct SCEVCollectStrides {
9007 : SE(SE), Strides(S) {}
9009 bool follow(
const SCEV *S) {
9014 bool isDone()
const {
return false; }
9018 struct SCEVCollectTerms {
9024 bool follow(
const SCEV *S) {
9025 if (isa<SCEVUnknown>(S) || isa<SCEVMulExpr>(S) ||
9026 isa<SCEVSignExtendExpr>(S)) {
9037 bool isDone()
const {
return false; }
9041 struct SCEVHasAddRec {
9042 bool &ContainsAddRec;
9044 SCEVHasAddRec(
bool &ContainsAddRec) : ContainsAddRec(ContainsAddRec) {
9045 ContainsAddRec =
false;
9048 bool follow(
const SCEV *S) {
9049 if (isa<SCEVAddRecExpr>(S)) {
9050 ContainsAddRec =
true;
9059 bool isDone()
const {
return false; }
9074 struct SCEVCollectAddRecMultiplies {
9079 : Terms(T), SE(SE) {}
9081 bool follow(
const SCEV *S) {
9082 if (
auto *Mul = dyn_cast<SCEVMulExpr>(S)) {
9083 bool HasAddRec =
false;
9085 for (
auto Op : Mul->operands()) {
9086 if (isa<SCEVUnknown>(Op)) {
9089 bool ContainsAddRec;
9090 SCEVHasAddRec ContiansAddRec(ContainsAddRec);
9092 HasAddRec |= ContainsAddRec;
9095 if (Operands.
size() == 0)
9109 bool isDone()
const {
return false; }
9120 SCEVCollectStrides StrideCollector(*
this, Strides);
9124 dbgs() <<
"Strides:\n";
9125 for (
const SCEV *S : Strides)
9126 dbgs() << *S <<
"\n";
9129 for (
const SCEV *S : Strides) {
9130 SCEVCollectTerms TermCollector(Terms);
9135 dbgs() <<
"Terms:\n";
9136 for (
const SCEV *
T : Terms)
9137 dbgs() << *
T <<
"\n";
9140 SCEVCollectAddRecMultiplies MulCollector(Terms, *
this);
9147 int Last = Terms.
size() - 1;
9148 const SCEV *Step = Terms[Last];
9152 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Step)) {
9154 for (
const SCEV *Op : M->operands())
9155 if (!isa<SCEVConstant>(Op))
9165 for (
const SCEV *&Term : Terms) {
9168 SCEVDivision::divide(SE, Term, Step, &Q, &R);
9179 remove_if(Terms, [](
const SCEV *E) {
return isa<SCEVConstant>(
E); }),
9182 if (Terms.size() > 0)
9193 for (
const SCEV *
T : Terms)
9201 if (
const SCEVMulExpr *Expr = dyn_cast<SCEVMulExpr>(S))
9202 return Expr->getNumOperands();
9207 if (isa<SCEVConstant>(T))
9210 if (isa<SCEVUnknown>(T))
9213 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(T)) {
9215 for (
const SCEV *Op : M->operands())
9216 if (!isa<SCEVConstant>(Op))
9229 Ty =
Store->getValueOperand()->getType();
9231 Ty =
Load->getType();
9241 const SCEV *ElementSize)
const {
9242 if (Terms.
size() < 1 || !ElementSize)
9251 dbgs() <<
"Terms:\n";
9252 for (
const SCEV *
T : Terms)
9253 dbgs() << *
T <<
"\n";
9257 std::sort(Terms.
begin(), Terms.
end());
9261 std::sort(Terms.
begin(), Terms.
end(), [](
const SCEV *LHS,
const SCEV *RHS) {
9269 for (
const SCEV *&Term : Terms) {
9271 SCEVDivision::divide(SE, Term, ElementSize, &Q, &R);
9279 for (
const SCEV *
T : Terms)
9284 dbgs() <<
"Terms after sorting:\n";
9285 for (
const SCEV *
T : NewTerms)
9286 dbgs() << *
T <<
"\n";
9289 if (NewTerms.
empty() ||
9299 dbgs() <<
"Sizes:\n";
9300 for (
const SCEV *S : Sizes)
9301 dbgs() << *S <<
"\n";
9313 if (
auto *AR = dyn_cast<SCEVAddRecExpr>(Expr))
9317 const SCEV *Res = Expr;
9318 int Last = Sizes.
size() - 1;
9319 for (
int i = Last;
i >= 0;
i--) {
9321 SCEVDivision::divide(*
this, Res, Sizes[
i], &Q, &R);
9324 dbgs() <<
"Res: " << *Res <<
"\n";
9325 dbgs() <<
"Sizes[i]: " << *Sizes[
i] <<
"\n";
9326 dbgs() <<
"Res divided by Sizes[i]:\n";
9327 dbgs() <<
"Quotient: " << *Q <<
"\n";
9328 dbgs() <<
"Remainder: " << *R <<
"\n";
9338 if (isa<SCEVAddRecExpr>(R)) {
9358 dbgs() <<
"Subscripts:\n";
9359 for (
const SCEV *S : Subscripts)
9360 dbgs() << *S <<
"\n";
9416 const SCEV *ElementSize) {
9433 if (Subscripts.
empty())
9437 dbgs() <<
"succeeded to delinearize " << *Expr <<
"\n";
9438 dbgs() <<
"ArrayDecl[UnknownSize]";
9439 for (
const SCEV *S : Sizes)
9440 dbgs() <<
"[" << *S <<
"]";
9442 dbgs() <<
"\nArrayRef";
9443 for (
const SCEV *S : Subscripts)
9444 dbgs() <<
"[" << *S <<
"]";
9453 void ScalarEvolution::SCEVCallbackVH::deleted() {
9454 assert(SE &&
"SCEVCallbackVH called with a null ScalarEvolution!");
9455 if (
PHINode *PN = dyn_cast<PHINode>(getValPtr()))
9456 SE->ConstantEvolutionLoopExitValue.erase(PN);
9461 void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(
Value *V) {
9462 assert(SE &&
"SCEVCallbackVH called with a null ScalarEvolution!");
9467 Value *Old = getValPtr();
9470 while (!Worklist.
empty()) {
9476 if (!Visited.
insert(U).second)
9478 if (
PHINode *PN = dyn_cast<PHINode>(U))
9479 SE->ConstantEvolutionLoopExitValue.erase(PN);
9484 if (
PHINode *PN = dyn_cast<PHINode>(Old))
9485 SE->ConstantEvolutionLoopExitValue.erase(PN);
9500 : F(F), TLI(TLI), AC(AC), DT(DT), LI(LI),
9502 WalkingBEDominatingConds(
false), ProvingSplitPredicate(
false),
9503 ValuesAtScopes(64), LoopDispositions(64), BlockDispositions(64),
9504 FirstUnknown(nullptr) {
9518 HasGuards = GuardDecl && !GuardDecl->use_empty();
9522 :
F(Arg.
F), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT),
9523 LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
9524 ValueExprMap(std::move(Arg.ValueExprMap)),
9525 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
9526 WalkingBEDominatingConds(
false), ProvingSplitPredicate(
false),
9527 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
9528 PredicatedBackedgeTakenCounts(
9529 std::move(Arg.PredicatedBackedgeTakenCounts)),
9530 ConstantEvolutionLoopExitValue(
9531 std::move(Arg.ConstantEvolutionLoopExitValue)),
9532 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
9533 LoopDispositions(std::move(Arg.LoopDispositions)),
9534 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
9535 BlockDispositions(std::move(Arg.BlockDispositions)),
9536 UnsignedRanges(std::move(Arg.UnsignedRanges)),
9537 SignedRanges(std::move(Arg.SignedRanges)),
9538 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
9539 UniquePreds(std::move(Arg.UniquePreds)),
9540 SCEVAllocator(std::move(Arg.SCEVAllocator)),
9541 FirstUnknown(Arg.FirstUnknown) {
9542 Arg.FirstUnknown =
nullptr;
9551 Tmp->~SCEVUnknown();
9553 FirstUnknown =
nullptr;
9555 ExprValueMap.
clear();
9556 ValueExprMap.
clear();
9561 for (
auto &BTCI : BackedgeTakenCounts)
9562 BTCI.second.clear();
9563 for (
auto &BTCI : PredicatedBackedgeTakenCounts)
9564 BTCI.second.clear();
9566 assert(PendingLoopPredicates.empty() &&
"isImpliedCond garbage");
9567 assert(!WalkingBEDominatingConds &&
"isLoopBackedgeGuardedByCond garbage!");
9568 assert(!ProvingSplitPredicate &&
"ProvingSplitPredicate garbage!");
9582 L->getHeader()->printAsOperand(OS,
false);
9586 L->getExitBlocks(ExitBlocks);
9587 if (ExitBlocks.
size() != 1)
9588 OS <<
"<multiple exits> ";
9593 OS <<
"Unpredictable backedge-taken count. ";
9598 L->getHeader()->printAsOperand(OS,
false);
9604 OS <<
", actual taken count either this or zero.";
9606 OS <<
"Unpredictable max backedge-taken count. ";
9611 L->getHeader()->printAsOperand(OS,
false);
9616 if (!isa<SCEVCouldNotCompute>(PBT)) {
9617 OS <<
"Predicated backedge-taken count is " << *PBT <<
"\n";
9618 OS <<
" Predicates:\n";
9621 OS <<
"Unpredictable predicated backedge-taken count. ";
9633 return "Computable";
9647 OS <<
"Classifying expressions for: ";
9656 if (!isa<SCEVCouldNotCompute>(SV)) {
9663 const Loop *L = LI.getLoopFor(I.getParent());
9669 if (!isa<SCEVCouldNotCompute>(AtUse)) {
9678 OS <<
"\t\t" "Exits: ";
9681 OS <<
"<<Unknown>>";
9689 OS <<
"\t\t" "LoopDispositions: { ";
9695 Iter->getHeader()->printAsOperand(OS,
false);
9703 OS <<
"\t\t" "LoopDispositions: { ";
9709 InnerL->getHeader()->printAsOperand(OS,
false);
9719 OS <<
"Determining loop execution counts for: ";
9728 auto &Values = LoopDispositions[S];
9729 for (
auto &V : Values) {
9730 if (V.getPointer() ==
L)
9735 auto &Values2 = LoopDispositions[S];
9736 for (
auto &V :
make_range(Values2.rbegin(), Values2.rend())) {
9737 if (V.getPointer() ==
L) {
9746 ScalarEvolution::computeLoopDisposition(
const SCEV *S,
const Loop *L) {
9747 switch (static_cast<SCEVTypes>(S->
getSCEVType())) {
9786 bool HasVarying =
false;
9787 for (
auto *Op : cast<SCEVNAryExpr>(S)->operands()) {
9812 if (
auto *I = dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue()))
9831 auto &Values = BlockDispositions[S];
9832 for (
auto &V : Values) {
9833 if (V.getPointer() == BB)
9838 auto &Values2 = BlockDispositions[S];
9839 for (
auto &V :
make_range(Values2.rbegin(), Values2.rend())) {
9840 if (V.getPointer() == BB) {
9849 ScalarEvolution::computeBlockDisposition(
const SCEV *S,
const BasicBlock *BB) {
9850 switch (static_cast<SCEVTypes>(S->
getSCEVType())) {
9898 dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) {
9924 void ScalarEvolution::forgetMemoizedResults(
const SCEV *S) {
9925 ValuesAtScopes.erase(S);
9926 LoopDispositions.erase(S);
9927 BlockDispositions.erase(S);
9928 UnsignedRanges.erase(S);
9929 SignedRanges.erase(S);
9930 ExprValueMap.
erase(S);
9933 auto RemoveSCEVFromBackedgeMap =
9935 for (
auto I = Map.begin(), E = Map.end(); I !=
E;) {
9936 BackedgeTakenInfo &BEInfo = I->second;
9937 if (BEInfo.hasOperand(S,
this)) {
9945 RemoveSCEVFromBackedgeMap(BackedgeTakenCounts);
9946 RemoveSCEVFromBackedgeMap(PredicatedBackedgeTakenCounts);
9954 while ((Pos = Str.find(From, Pos)) != std::string::npos) {
9963 std::string &S = Map[
L];
9987 VerifyMap BackedgeDumpsOld, BackedgeDumpsNew;
10000 "New loops suddenly appeared!");
10003 OldE = BackedgeDumpsOld.
end(),
10004 NewI = BackedgeDumpsNew.
begin();
10005 OldI != OldE; ++OldI, ++NewI) {
10006 assert(OldI->first == NewI->first &&
"Loop order changed!");
10013 if (OldI->second != NewI->second &&
10014 OldI->second.find(
"undef") == std::string::npos &&
10015 NewI->second.find(
"undef") == std::string::npos &&
10016 OldI->second !=
"***COULDNOTCOMPUTE***" &&
10017 NewI->second !=
"***COULDNOTCOMPUTE***") {
10018 dbgs() <<
"SCEVValidator: SCEV for loop '"
10019 << OldI->first->getHeader()->getName()
10020 <<
"' changed from '" << OldI->second
10021 <<
"' to '" << NewI->second <<
"'!\n";
10058 "Scalar Evolution Analysis",
false,
true)
10065 char ScalarEvolutionWrapperPass::ID = 0;
10073 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
10074 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
10075 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
10076 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
10109 void *IP =
nullptr;
10110 if (
const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
10114 UniquePreds.InsertNode(Eq, IP);
10126 void *IP =
nullptr;
10127 if (
const auto *S = UniquePreds.FindNodeOrInsertPos(ID, IP))
10129 auto *OF =
new (SCEVAllocator)
10131 UniquePreds.InsertNode(OF, IP);
10150 SCEVPredicateRewriter
Rewriter(L, SE, NewPreds, Pred);
10162 for (
auto *Pred : ExprPreds)
10163 if (
const auto *IPred = dyn_cast<SCEVEqualPredicate>(Pred))
10164 if (IPred->getLHS() == Expr)
10165 return IPred->getRHS();
10174 if (AR && AR->getLoop() == L && AR->isAffine()) {
10177 const SCEV *Step = AR->getStepRecurrence(SE);
10182 AR->getNoWrapFlags());
10190 if (AR && AR->getLoop() == L && AR->isAffine()) {
10193 const SCEV *Step = AR->getStepRecurrence(SE);
10198 AR->getNoWrapFlags());
10209 return Pred && Pred->
implies(A);
10223 return SCEVPredicateRewriter::rewrite(S, L, *
this,
nullptr, &Preds);
10231 S = SCEVPredicateRewriter::rewrite(S, L, *
this, &TransformPreds,
nullptr);
10239 for (
auto *
P : TransformPreds)
10248 : FastID(ID), Kind(Kind) {}
10261 return Op->LHS == LHS && Op->RHS == RHS;
10269 OS.
indent(Depth) <<
"Equal predicate: " << *LHS <<
" == " << *RHS <<
"\n";
10282 return Op && Op->AR == AR &&
setFlags(Flags, Op->Flags) == Flags;
10318 if (Step->getValue()->getValue().isNonNegative())
10322 return ImpliedFlags;
10336 auto I = SCEVToPreds.
find(Expr);
10337 if (I == SCEVToPreds.
end())
10343 if (
const auto *Set = dyn_cast<SCEVUnionPredicate>(N))
10344 return all_of(Set->Preds,
10345 [
this](
const SCEVPredicate *I) {
return this->implies(I); });
10347 auto ScevPredsIt = SCEVToPreds.find(
N->getExpr());
10348 if (ScevPredsIt == SCEVToPreds.end())
10350 auto &SCEVPreds = ScevPredsIt->second;
10352 return any_of(SCEVPreds,
10359 for (
auto Pred : Preds)
10360 Pred->
print(OS, Depth);
10364 if (
const auto *Set = dyn_cast<SCEVUnionPredicate>(N)) {
10365 for (
auto Pred : Set->Preds)
10374 assert(Key &&
"Only SCEVUnionPredicate doesn't have an "
10375 " associated expression!");
10377 SCEVToPreds[Key].push_back(N);
10378 Preds.push_back(N);
10383 : SE(SE), L(L), Generation(0), BackedgeCount(nullptr) {}
10387 RewriteEntry &Entry = RewriteMap[Expr];
10390 if (Entry.second && Generation == Entry.first)
10391 return Entry.second;
10396 Expr = Entry.second;
10399 Entry = {Generation, NewSCEV};
10405 if (!BackedgeCount) {
10410 return BackedgeCount;
10417 updateGeneration();
10424 void PredicatedScalarEvolution::updateGeneration() {
10426 if (++Generation == 0) {
10427 for (
auto &II : RewriteMap) {
10428 const SCEV *Rewritten = II.second.second;
10437 const auto *AR = cast<SCEVAddRecExpr>(Expr);
10445 auto II = FlagsMap.insert({V, Flags});
10453 const auto *AR = cast<SCEVAddRecExpr>(Expr);
10458 auto II = FlagsMap.find(V);
10460 if (II != FlagsMap.end())
10474 for (
auto *
P : NewPreds)
10477 updateGeneration();
10478 RewriteMap[SE.
getSCEV(V)] = {Generation, New};
10484 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds),
10485 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {
10486 for (
const auto &I : Init.FlagsMap)
10487 FlagsMap.insert(I);
10493 for (
auto &I : *BB) {
10498 auto II = RewriteMap.
find(Expr);
10500 if (II == RewriteMap.
end())
10504 if (II->second.second == Expr)
10507 OS.
indent(Depth) <<
"[PSE]" << I <<
":\n";
10508 OS.
indent(Depth + 2) << *Expr <<
"\n";
10509 OS.
indent(Depth + 2) <<
"--> " << *II->second.second <<
"\n";
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask=NoWrapMask) const
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
const SCEV * getTruncateOrNoop(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
const Use & getOperandUse(unsigned i) const
virtual const SCEV * getExpr() const =0
Returns the SCEV to which this predicate applies, or nullptr if this is a SCEVUnionPredicate.
const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
Return the value of this chain of recurrences at the specified iteration number.
static bool containsParameters(SmallVectorImpl< const SCEV * > &Terms)
void AddPointer(const void *Ptr)
Add* - Add various data types to Bit data.
IncrementWrapFlags getFlags() const
Returns the set assumed no overflow flags.
APInt multiplicativeInverse(const APInt &modulo) const
Computes the multiplicative inverse of this APInt for a given modulo.
void computeKnownBits(const Value *V, APInt &KnownZero, APInt &KnownOne, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
Type * getSourceElementType() const
void push_back(const T &Elt)
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
This routine provides some synthesis utilities to produce sequences of values.
A parsed version of the target data layout string in and methods for querying it. ...
static cl::opt< bool > VerifySCEV("verify-scev", cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"))
static ConstantInt * getFalse(LLVMContext &Context)
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
void setNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Proves that V doesn't overflow by adding SCEV predicate.
This class is the base class for the comparison instructions.
The SCEV properly dominates the block.
bool invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Trigger the invalidation of some other analysis pass if not already handled and return whether it was...
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
static const SCEV * SolveLinEquationWithOverflow(const APInt &A, const APInt &B, ScalarEvolution &SE)
Finds the minimum unsigned root of the following equation:
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
const SCEV * getGEPExpr(GEPOperator *GEP, const SmallVectorImpl< const SCEV * > &IndexExprs)
Returns an expression for a GEP.
SCEVCastExpr(const FoldingSetNodeIDRef ID, unsigned SCEVTy, const SCEV *op, Type *ty)
uint64_t getZExtValue() const
Get zero extended value.
bool isOne() const
Return true if the expression is a constant one.
const SCEV * getExitCount(Loop *L, BasicBlock *ExitingBlock)
Get the expression for the number of loop iterations for which this loop is guaranteed not to exit vi...
const SCEV * getConstant(ConstantInt *V)
STATISTIC(NumFunctions,"Total number of functions")
static const SCEV * getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE)
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
ConstantRange sextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
APInt GreatestCommonDivisor(const APInt &Val1, const APInt &Val2)
Compute GCD of two APInt values.
static const SCEV * removeConstantFactors(ScalarEvolution &SE, const SCEV *T)
bool canConstantFoldCallTo(const Function *F)
canConstantFoldCallTo - Return true if its even possible to fold a call to the specified function...
A Module instance is used to store all the information related to an LLVM module. ...
LLVMContext & getContext() const
static void PushDefUseChildren(Instruction *I, SmallVectorImpl< Instruction * > &Worklist)
Push users of the given Instruction onto the given Worklist.
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
SCEVUnionPredicate()
Union predicates don't get cached so create a dummy set ID for it.
bool isZero() const
Return true if the expression is a constant zero.
static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge, Value *&C, Value *&LHS, Value *&RHS)
virtual bool implies(const SCEVPredicate *N) const =0
Returns true if this predicate implies N.
void setBit(unsigned bitPosition)
Set a given bit to 1.
bool isKnownNotFullPoison(const Instruction *PoisonI)
Return true if this function can prove that if PoisonI is executed and yields a full-poison value (al...
unsigned getNumOperands() const
static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow)
Compute the result of "n choose k", the binomial coefficient.
const SCEV * getPointerBase(const SCEV *V)
Transitively follow the chain of pointer-type operands until reaching a SCEV that does not have a sin...
The main scalar evolution driver.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
This class represents a function call, abstracting a target machine's calling convention.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
IncrementWrapFlags
Similar to SCEV::NoWrapFlags, but with slightly different semantics for FlagNUSW. ...
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
An immutable pass that tracks lazily created AssumptionCache objects.
scalar Scalar Evolution false
bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
static const SCEV * getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty, ScalarEvolution *SE)
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass...
bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
bool mayHaveSideEffects() const
Return true if the instruction may have side effects.
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, Optional< unsigned > InRangeIndex=None, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static LLVM_NODISCARD SCEV::NoWrapFlags setFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OnFlags)
std::vector< LoopT * >::const_reverse_iterator reverse_iterator
bool isSigned() const
Determine if this instruction is using a signed comparison.
A cache of .assume calls within a function.
static void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
bool properlyDominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV properly dominate the specified basic block...
static void PushLoopPHIs(const Loop *L, SmallVectorImpl< Instruction * > &Worklist)
Push PHI nodes in the header of the given loop onto the given Worklist.
uint32_t GetMinTrailingZeros(const SCEV *S)
Determine the minimum number of zero bits that S is guaranteed to end in (at every loop iteration)...
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop...
This class represents a truncation of an integer value to a smaller integer value.
bool isMask(unsigned numBits, const APInt &APIVal)
INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass,"scalar-evolution","Scalar Evolution Analysis", false, true) INITIALIZE_PASS_END(ScalarEvolutionWrapperPass
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
static ConstantInt * EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C, ScalarEvolution &SE)
LoopT * getParentLoop() const
const Function * getParent() const
Return the enclosing method, or null if none.
static const SCEV * getSignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether the backedge of the loop is protected by a conditional between LHS and RHS...
Analysis pass which computes a DominatorTree.
An instruction for reading from memory.
static std::pair< const SCEV *, ConstantInt * > splitAddExpr(const SCEV *S)
Try to split a SCEVAddExpr into a pair of {SCEV, ConstantInt}.
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
static ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
An object of this class is returned by queries that could not be answered.
void reserve(size_type N)
static bool HasSameValue(const SCEV *A, const SCEV *B)
SCEV structural equivalence is usually sufficient for testing whether two expressions are equal...
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
const std::vector< BlockT * > & getBlocks() const
Get a list of the basic blocks which make up this loop.
const SCEV * getUMaxFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS)
Promote the operands to the wider of the types using zero-extension, and then perform a umax operatio...
const SCEV * getExpr() const override
Returns the SCEV to which this predicate applies, or nullptr if this is a SCEVUnionPredicate.
static Optional< ConstantRange > GetRangeFromMetadata(Value *V)
Helper method to assign a range to V from metadata present in the IR.
bool hasNoSelfWrap() const
BlockT * getHeader() const
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
NodeT * getIDom(NodeT *BB) const
ConstantInt * findCaseDest(BasicBlock *BB)
Finds the unique case value for a given successor.
This is the base class for unary cast operator classes.
bool propagatesFullPoison(const Instruction *I)
Return true if this function can prove that I is guaranteed to yield full-poison (all bits poison) if...
ConstantRange smax(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a signed maximum of a value in thi...
bool isOffsetOf(Type *&STy, Constant *&FieldNo) const
const SCEV * getStart() const
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
iterator begin()
Instruction iterator methods.
static Constant * getAdd(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
bool isKnownNonPositive(const SCEV *S)
Test if the given expression is known to be non-positive.
Predicate getSignedPredicate()
For example, ULT->SLT, ULE->SLE, UGT->SGT, UGE->SGE, SLT->Failed assert.
static void getLoopBackedgeTakenCounts(Loop *L, VerifyMap &Map, ScalarEvolution &SE)
getLoopBackedgeTakenCounts - Helper method for verifyAnalysis.
const SCEV * getZero(Type *Ty)
Return a SCEV for the constant 0 of a specific type.
bool isNegative() const
Determine sign of this APInt.
The SCEV is loop-invariant.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static LLVM_NODISCARD SCEVWrapPredicate::IncrementWrapFlags setFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OnFlags)
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
bool match(Val *V, const Pattern &P)
#define INITIALIZE_PASS_DEPENDENCY(depName)
void computeAccessFunctions(const SCEV *Expr, SmallVectorImpl< const SCEV * > &Subscripts, SmallVectorImpl< const SCEV * > &Sizes)
Return in Subscripts the access functions for each dimension in Sizes (third step of delinearization)...
bool isUnconditional() const
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
ArrayRef< const SCEVPredicate * > getPredicatesForExpr(const SCEV *Expr)
Returns a reference to a vector containing all predicates which apply to Expr.
static Constant * getIntegerCast(Constant *C, Type *Ty, bool isSigned)
Create a ZExt, Bitcast or Trunc for integer -> integer casts.
const SCEVPredicate * getWrapPredicate(const SCEVAddRecExpr *AR, SCEVWrapPredicate::IncrementWrapFlags AddedFlags)
bool isAlignOf(Type *&AllocTy) const
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
bool isIdenticalTo(const Instruction *I) const
Return true if the specified instruction is exactly identical to the current one. ...
ConstantRange truncate(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly smaller than the current typ...
struct fuzzer::@269 Flags
const APInt & getValue() const
Return the constant as an APInt value reference.
const DataLayout & getDataLayout() const
Return the DataLayout associated with the module this SCEV instance is operating on.
Class to represent struct types.
void verifyAnalysis() const override
verifyAnalysis() - This member can be implemented by a analysis pass to check state of analysis infor...
A Use represents the edge between a Value definition and its users.
bool hasNoSignedWrap() const
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
const SCEV *const * op_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
static Constant * AddOne(Constant *C)
Add one to a Constant.
bool Eq(const uint8_t *Data, size_t Size, const char *Str)
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
uint64_t getTypeSizeInBits(Type *Ty) const
Return the size in bits of the specified type, for which isSCEVable must return true.
static int CompareValueComplexity(SmallSet< std::pair< Value *, Value * >, 8 > &EqCache, const LoopInfo *const LI, Value *LV, Value *RV, unsigned Depth)
Compare the two values LV and RV in terms of their "complexity" where "complexity" is a partial (and ...
void AddInteger(signed I)
static Constant * getSExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This node is the base class for n'ary commutative operators.
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
Analysis pass that exposes the LoopInfo for a function.
ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
op_iterator op_begin() const
void setNoWrapFlags(NoWrapFlags Flags)
Set flags for a non-recurrence without clearing previously set flags.
This node represents multiplication of some number of SCEVs.
void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
ConstantRange multiply(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
LLVM_NODISCARD bool empty() const
APInt lshr(const APInt &LHS, unsigned shiftAmt)
Logical right-shift function.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static const SCEV * BinomialCoefficient(const SCEV *It, unsigned K, ScalarEvolution &SE, Type *ResultTy)
Compute BC(It, K). The result has width W. Assume, K > 0.
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
APInt shl(unsigned shiftAmt) const
Left-shift function.
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
static Constant * getZExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS...
reverse_iterator rbegin() const
static bool isPrivateLinkage(LinkageTypes Linkage)
A constant value that is initialized with an expression using other constant values.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
const SCEV * getOffsetOfExpr(Type *IntTy, StructType *STy, unsigned FieldNo)
Return an expression for offsetof on the given field with type IntTy.
unsigned getSmallConstantMaxTripCount(Loop *L)
Returns the upper bound of the loop trip count as a normal unsigned value.
ConstantRange getSignedRange(const SCEV *S)
Determine the signed range for a particular SCEV.
This node represents a polynomial recurrence on the trip count of the specified loop.
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
bool implies(const SCEVPredicate *N) const override
Implementation of the SCEVPredicate interface.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
const T & getValue() const LLVM_LVALUE_FUNCTION
APInt udiv(const APInt &LHS, const APInt &RHS)
Unsigned division function for APInt.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
bool sgt(const APInt &RHS) const
Signed greather than comparison.
BasicBlock * getSuccessor(unsigned i) const
const SCEV * getSizeOfExpr(Type *IntTy, Type *AllocTy)
Return an expression for sizeof AllocTy that is type IntTy.
const SCEVAddRecExpr * getAsAddRec(Value *V)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
Base class for the actual dominator tree node.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
unsigned getActiveBits() const
Compute the number of active bits in the value.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
APInt sdiv(const APInt &LHS, const APInt &RHS)
Signed division function for APInt.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
An instruction for storing to memory.
static cl::opt< unsigned > MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden, cl::desc("Maximum number of iterations SCEV will ""symbolically execute a constant ""derived loop"), cl::init(100))
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
bool isArrayTy() const
True if this is an instance of ArrayType.
SelectClass_match< Cond, LHS, RHS > m_Select(const Cond &C, const LHS &L, const RHS &R)
static Constant * getUDiv(Constant *C1, Constant *C2, bool isExact=false)
APInt ashr(const APInt &LHS, unsigned shiftAmt)
Arithmetic right-shift function.
Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
const SCEV * getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, SCEV::NoWrapFlags Flags)
Get an add recurrence expression for the specified loop.
size_t getNumOperands() const
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
Class to represent pointers.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static ConstantRange makeSatisfyingICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the largest range such that all values in the returned range satisfy the given predicate with...
static cl::opt< unsigned > MaxIterations("max-cg-scc-iterations", cl::ReallyHidden, cl::init(4))
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getNumIncomingValues() const
Return the number of incoming edges.
bool implies(const SCEVPredicate *N) const override
Returns true if this predicate implies N.
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
bool isLoopEntryGuardedByCond(const Loop *L, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test whether entry to the loop is protected by a conditional between LHS and RHS. ...
static const SCEV * MatchNotExpr(const SCEV *Expr)
If Expr computes ~A, return A else return nullptr.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
ValTy * getReturnedArgOperand() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
const SCEV * getCouldNotCompute()
Visit all nodes in the expression tree using worklist traversal.
ScalarEvolution run(Function &F, FunctionAnalysisManager &AM)
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
initializer< Ty > init(const Ty &Val)
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
bool erase(const KeyT &Val)
static bool isInternalLinkage(LinkageTypes Linkage)
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values...
bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
Subclasses of this class are all able to terminate a basic block.
A set of analyses that are preserved following a run of a transformation pass.
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt...
static cl::opt< bool > VerifySCEVMap("verify-scev-maps", cl::desc("Verify no dangling value in ScalarEvolution's ""ExprValueMap (slow)"))
ConstantRange intersectWith(const ConstantRange &CR) const
Return the range that results from the intersection of this range with another range.
This class defines a simple visitor class that may be used for various SCEV analysis purposes...
bool isSizeOf(Type *&AllocTy) const
APInt trunc(unsigned width) const
Truncate to new width.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type. ...
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
LLVM Basic Block Representation.
This class represents a binary unsigned division operation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
Type * getType() const
Return the LLVM type of this SCEV expression.
LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)
Return the "disposition" of the given SCEV with respect to the given loop.
static void GroupByComplexity(SmallVectorImpl< const SCEV * > &Ops, LoopInfo *LI)
Given a list of SCEV objects, order them by their complexity, and group objects of the same complexit...
Conditional or Unconditional Branch instruction.
bool containsAddRecurrence(const SCEV *S)
Return true if the SCEV is a scAddRecExpr or it contains scAddRecExpr.
bool sge(const APInt &RHS) const
Signed greather or equal comparison.
static bool IsMinConsistingOf(ScalarEvolution &SE, const SCEV *MaybeMinExpr, const SCEV *Candidate)
Is MaybeMinExpr an SMin or UMin of Candidate and some other values?
bool isEquality() const
Return true if this predicate is either EQ or NE.
static int CompareSCEVComplexity(SmallSet< std::pair< const SCEV *, const SCEV * >, 8 > &EqCacheSCEV, const LoopInfo *const LI, const SCEV *LHS, const SCEV *RHS, unsigned Depth=0)
This is an important base class in LLVM.
bool isMaxSignedValue() const
Determine if this is the largest signed value.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
const SCEV * getOperand(unsigned i) const
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * Allocate(size_t Size, size_t Alignment)
Allocate space at the specified alignment.
static unsigned getConstantTripCount(const SCEVConstant *ExitCount)
SCEVWrapPredicate(const FoldingSetNodeIDRef ID, const SCEVAddRecExpr *AR, IncrementWrapFlags Flags)
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
const SCEV * getExpr() const override
Implementation of the SCEVPredicate interface.
bool hasNoSignedWrap() const
Determine whether the no signed wrap flag is set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
static cl::opt< unsigned > MulOpsInlineThreshold("scev-mulops-inline-threshold", cl::Hidden, cl::desc("Threshold for inlining multiplication operands into a SCEV"), cl::init(1000))
void add(const SCEVPredicate *N)
Adds a predicate to this union.
bool isZeroValue() const
Return true if the value is negative zero or null value.
void print(raw_ostream &OS, unsigned Depth) const override
Prints a textual representation of this predicate with an indentation of Depth.
static bool canConstantEvolve(Instruction *I, const Loop *L)
Determine whether this instruction can constant evolve within this loop assuming its operands can all...
void visitAll(const SCEV *Root)
const SCEV * getSMaxExpr(const SCEV *LHS, const SCEV *RHS)
bool isOverflowIntrinsicNoWrap(const IntrinsicInst *II, const DominatorTree &DT)
Returns true if the arithmetic part of the II 's result is used only along the paths control dependen...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt sext(unsigned width) const
Sign extend to a new width.
const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
Represent the analysis usage information of a pass.
ConstantRange udiv(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an unsigned division of a value in...
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
PreservedAnalysisChecker getChecker() const
Build a checker for this PreservedAnalyses and the specified analysis type.
bool isFalseWhenEqual() const
This is just a convenience.
bool hasOperand(const SCEV *S, const SCEV *Op) const
Test whether the given SCEV has Op as a direct or indirect operand.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
This instruction compares its operands according to the predicate given to the constructor.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return LHS-RHS. Minus is represented in SCEV as A+B*-1.
const SCEV * getMaxBackedgeTakenCount(const Loop *L)
Similar to getBackedgeTakenCount, except return the least SCEV value that is known never to be less t...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Utility class for integer arithmetic operators which may exhibit overflow - Add, Sub, and Mul.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
unsigned getBitWidth() const
Return the number of bits in the APInt.
static const unsigned End
const SCEV * rewriteUsingPredicate(const SCEV *S, const Loop *L, SCEVUnionPredicate &A)
Re-writes the SCEV according to the Predicates in A.
bool SCEVExprContains(const SCEV *Root, PredTy Pred)
Return true if any node in Root satisfies the predicate Pred.
op_range operands() const
const SCEVAddRecExpr * getPostIncExpr(ScalarEvolution &SE) const
Return an expression representing the value of this expression one iteration of the loop ahead...
const SCEV * getPredicatedBackedgeTakenCount(const Loop *L, SCEVUnionPredicate &Predicates)
Similar to getBackedgeTakenCount, except it will add a set of SCEV predicates to Predicates that are ...
Value * getPointerOperand()
FunctionPass class - This class is used to implement most global optimizations.
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
bool isMaxValue() const
Determine if this is the largest unsigned value.
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Value * getOperand(unsigned i) const
unsigned getSmallConstantTripCount(Loop *L)
Returns the maximum trip count of the loop if it is a single-exit loop and we can compute a small max...
static Constant * getICmp(unsigned short pred, Constant *LHS, Constant *RHS, bool OnlyIfReduced=false)
get* - Return some common constants without having to specify the full Instruction::OPCODE identifier...
iterator_range< block_iterator > blocks()
void print(raw_ostream &OS) const
Print out the bounds to a stream.
Class to represent integer types.
static LLVM_NODISCARD SCEV::NoWrapFlags clearFlags(SCEV::NoWrapFlags Flags, SCEV::NoWrapFlags OffFlags)
Predicate getPredicate() const
Return the predicate for this instruction.
ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
bool isKnownNegative(const SCEV *S)
Test if the given expression is known to be negative.
static Constant * getNot(Constant *C)
void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
MutableArrayRef< WeakVH > assumptions()
Access the list of assumption handles currently tracked for this function.
void setNoWrapFlags(NoWrapFlags Flags)
Set flags for a recurrence without clearing any previously set flags.
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL)
ConstantFoldLoadFromConstPtr - Return the value that a load from C would produce if it is constant an...
static Constant * getAllOnesValue(Type *Ty)
Get the all ones value.
const SCEV * getLHS() const
const APInt & getAPInt() const
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
void findArrayDimensions(SmallVectorImpl< const SCEV * > &Terms, SmallVectorImpl< const SCEV * > &Sizes, const SCEV *ElementSize) const
Compute the array dimensions Sizes from the set of Terms extracted from the memory access function of...
bool isEmptySet() const
Return true if this set contains no members.
const SCEV * getUMinFromMismatchedTypes(const SCEV *LHS, const SCEV *RHS)
Promote the operands to the wider of the types using zero-extension, and then perform a umin operatio...
static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S, BasicBlock *BB)
bool isPointerTy() const
True if this is an instance of PointerType.
iterator erase(const_iterator CI)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
bool isNonConstantNegative() const
Return true if the specified scev is negated, but not a constant.
const SCEVAddRecExpr * convertSCEVToAddRecWithPredicates(const SCEV *S, const Loop *L, SmallPtrSetImpl< const SCEVPredicate * > &Preds)
Tries to convert the S expression to an AddRec expression, adding additional predicates to Preds as r...
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
void print(raw_ostream &OS, unsigned Depth) const
Print the SCEV mappings done by the Predicated Scalar Evolution.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
const SCEV * getNoopOrZeroExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldCompareInstOperands - Attempt to constant fold a compare instruction (icmp/fcmp) with the...
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const SCEV * getRHS() const
const SCEV * getUDivExactExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
bool isRelational() const
Return true if the predicate is relational (not EQ or NE).
The SCEV is loop-variant (unknown).
This class represents an assumption made using SCEV expressions which can be checked at run-time...
static Constant * EvaluateExpression(Value *V, const Loop *L, DenseMap< Instruction *, Constant * > &Vals, const DataLayout &DL, const TargetLibraryInfo *TLI)
EvaluateExpression - Given an expression that passes the getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node in the loop has the value PHIVal.
void dump() const
This method is used for debugging.
static cl::opt< unsigned > MaxSCEVCompareDepth("scalar-evolution-max-scev-compare-depth", cl::Hidden, cl::desc("Maximum depth of recursive SCEV complexity comparisons"), cl::init(32))
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
bool isConditional() const
const SCEV * getSMinExpr(const SCEV *LHS, const SCEV *RHS)
RetVal visitCouldNotCompute(const SCEVCouldNotCompute *S)
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
A function analysis which provides an AssumptionCache.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
static int numberOfTerms(const SCEV *S)
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
bool hasNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Returns true if we've proved that V doesn't wrap by means of a SCEV predicate.
Iterator for intrusive lists based on ilist_node.
static void Merge(const std::string &Input, const std::vector< std::string > Result, size_t NumNewFeatures)
static PHINode * getConstantEvolvingPHI(Value *V, const Loop *L)
getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node in the loop that V is deri...
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2)
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the generic address space (address sp...
This is the shared class of boolean and integer constants.
bool isAlwaysTrue() const override
Implementation of the SCEVPredicate interface.
bool isKnownPositive(const SCEV *S)
Test if the given expression is known to be positive.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Return the number of times the sign bit of the register is replicated into the other bits...
bool slt(const APInt &RHS) const
Signed less than comparison.
bool hasNoUnsignedWrap() const
Determine whether the no unsigned wrap flag is set.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
void initializeScalarEvolutionWrapperPassPass(PassRegistry &)
reverse_iterator rend() const
const SCEV * getTruncateExpr(const SCEV *Op, Type *Ty)
static LLVM_NODISCARD SCEVWrapPredicate::IncrementWrapFlags clearFlags(SCEVWrapPredicate::IncrementWrapFlags Flags, SCEVWrapPredicate::IncrementWrapFlags OffFlags)
Convenient IncrementWrapFlags manipulation methods.
const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
void delinearize(const SCEV *Expr, SmallVectorImpl< const SCEV * > &Subscripts, SmallVectorImpl< const SCEV * > &Sizes, const SCEV *ElementSize)
Split this SCEVAddRecExpr into two vectors of SCEVs representing the subscripts and sizes of an array...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
bool isAllOnesValue() const
Return true if the expression is a constant all-ones value.
This is a utility class that provides an abstraction for the common functionality between Instruction...
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
The SCEV dominates the block.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
unsigned getSmallConstantTripMultiple(Loop *L)
Returns the largest constant divisor of the trip count of the loop if it is a single-exit loop and we...
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
This class represents a range of values.
bool isVolatile() const
Return true if this is a load from a volatile memory location.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
const APInt & getLower() const
Return the lower value for this range.
INITIALIZE_PASS(HexagonGenMux,"hexagon-mux","Hexagon generate mux instructions", false, false) void HexagonGenMux I isValid()
bool isTrueWhenEqual() const
This is just a convenience.
LLVM_NODISCARD T pop_back_val()
static LLVM_NODISCARD SCEVWrapPredicate::IncrementWrapFlags getImpliedFlags(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Returns the set of SCEVWrapPredicate no wrap flags implied by a SCEVAddRecExpr.
CHAIN = SC CHAIN, Imm128 - System call.
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
ScalarEvolution(Function &F, TargetLibraryInfo &TLI, AssumptionCache &AC, DominatorTree &DT, LoopInfo &LI)
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
static Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
iterator_range< detail::value_sequence_iterator< ValueT > > seq(ValueT Begin, ValueT End)
ConstantInt * getValue() const
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
const SCEV * getUMaxExpr(const SCEV *LHS, const SCEV *RHS)
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void print(raw_ostream &OS, const Module *=nullptr) const override
print - Print out the internal state of the pass.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
scalar Scalar Evolution Analysis
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static Constant * getOtherIncomingValue(PHINode *PN, BasicBlock *BB)
LoopDisposition
An enum describing the relationship between a SCEV and a loop.
Class for arbitrary precision integers.
This node represents an addition of some number of SCEVs.
const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty)
SetVector< ValueOffsetPair > * getSCEVValues(const SCEV *S)
Return the Value set from which the SCEV expr is generated.
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class represents a signed maximum selection.
iterator_range< user_iterator > users()
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
static const SCEV * getUnsignedOverflowLimitForStep(const SCEV *Step, ICmpInst::Predicate *Pred, ScalarEvolution *SE)
DenseMap< const Loop *, std::string > VerifyMap
static Constant * getCast(unsigned ops, Constant *C, Type *Ty, bool OnlyIfReduced=false)
Convenience function for getting a Cast operation.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical add expression, or something simpler if possible.
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
void visitAll(const SCEV *Root, SV &Visitor)
Use SCEVTraversal to visit all nodes in the given expression tree.
static void clear(coro::Shape &Shape)
Value * getCondition() const
iterator insert(iterator I, T &&Elt)
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
A utility class that uses RAII to save and restore the value of a variable.
bool isMinValue() const
Determine if this is the smallest unsigned value.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
SCEVPredicate(const SCEVPredicate &)=default
static Constant * BuildConstantFromSCEV(const SCEV *V)
This builds up a Constant using the ConstantExpr interface.
bool isStructTy() const
True if this is an instance of StructType.
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Analysis pass that exposes the ScalarEvolution for a function.
ConstantRange umax(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an unsigned maximum of a value in ...
This class represents a zero extension of a small integer value to a larger integer value...
Virtual Register Rewriter
static Constant * getNeg(Constant *C, bool HasNUW=false, bool HasNSW=false)
static Optional< std::pair< const SCEVConstant *, const SCEVConstant * > > SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE)
Find the roots of the quadratic equation for the given quadratic chrec {L,+,M,+,N}.
const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
static void replaceSubString(std::string &Str, StringRef From, StringRef To)
replaceSubString - Replaces all occurrences of From in Str with To.
const SCEVUnionPredicate & getUnionPredicate() const
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
void collectParametricTerms(const SCEV *Expr, SmallVectorImpl< const SCEV * > &Terms)
Collect parametric terms occurring in step expressions (first step of delinearization).
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Value * getCondition() const
void emplace_back(ArgTypes &&...Args)
void ComputeSignBit(const Value *V, bool &KnownZero, bool &KnownOne, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Determine whether the sign bit is known to be zero or one.
BasicBlock * getDefaultDest() const
The SCEV does not dominate the block.
FoldingSetNodeIDRef - This class describes a reference to an interned FoldingSetNodeID, which can be a useful to store node id data rather than using plain FoldingSetNodeIDs, since the 32-element SmallVector is often much larger than necessary, and the possibility of heap allocation means it requires a non-trivial destructor call.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
pointer data()
Return a pointer to the vector's buffer, even if empty().
This class represents an analyzed expression in the program.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isInBounds() const
Test whether this is an inbounds GEP, as defined by LangRef.html.
unsigned greater or equal
Represents a single loop in the control flow graph.
static PHINode * getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L, DenseMap< Instruction *, PHINode * > &PHIMap)
getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by recursing through each instructi...
static Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
This class represents a sign extension of a small integer value to a larger integer value...
iterator find(const KeyT &Val)
bool isSignBit() const
Check if the APInt's value is returned by getSignBit.
BlockDisposition
An enum describing the relationship between a SCEV and a basic block.
bool isKnownNonZero(const SCEV *S)
Test if the given expression is known to be non-zero.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
BlockT * getLoopPredecessor() const
If the given loop's header has exactly one unique predecessor outside the loop, return it...
This class represents an unsigned maximum selection.
static LLVM_NODISCARD SCEV::NoWrapFlags maskFlags(SCEV::NoWrapFlags Flags, int Mask)
Convenient NoWrapFlags manipulation that hides enum casts and is visible in the ScalarEvolution name ...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static bool CollectAddOperandsWithScales(DenseMap< const SCEV *, APInt > &M, SmallVectorImpl< const SCEV * > &NewOps, APInt &AccumulatedConstant, const SCEV *const *Ops, size_t NumOperands, const APInt &Scale, ScalarEvolution &SE)
Process the given Ops list, which is a list of operands to be added under the given scale...
AnalysisUsage & addRequiredTransitive()
const Loop * getLoop() const
bool isGuaranteedToExecuteForEveryIteration(const Instruction *I, const Loop *L)
Return true if this function can prove that the instruction I is executed for every iteration of the ...
void print(raw_ostream &OS) const
OutputIt transform(R &&Range, OutputIt d_first, UnaryPredicate P)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere...
static StringRef loopDispositionToStr(ScalarEvolution::LoopDisposition LD)
API to communicate dependencies between analyses during invalidation.
iterator find_as(const LookupKeyT &Val)
Alternate version of find() which allows a different, and possibly less expensive, key type.
Analysis pass providing the TargetLibraryInfo.
iterator_range< df_iterator< T > > depth_first(const T &G)
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
const SCEVPredicate * getEqualPredicate(const SCEVUnknown *LHS, const SCEVConstant *RHS)
const SCEV * getBackedgeTakenCount(const Loop *L)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
bool isUnsigned() const
Determine if this instruction is using an unsigned comparison.
const APInt & getUpper() const
Return the upper value for this range.
bool isSingleEdge() const
static ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, const ConstantRange &Other, unsigned NoWrapKind)
Return the largest range containing all X such that "X BinOpC Y" is guaranteed not to wrap (overflow)...
bool SimplifyICmpOperands(ICmpInst::Predicate &Pred, const SCEV *&LHS, const SCEV *&RHS, unsigned Depth=0)
Simplify LHS and RHS in a comparison with predicate Pred.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This templated class represents "all analyses that operate over \<a particular IR unit\>" (e...
user_iterator user_begin()
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
This class represents a composition of other SCEV predicates, and is the class that most clients will...
Constant * ConstantFoldInstOperands(Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands...
unsigned getSCEVType() const
const SCEV * getUMinExpr(const SCEV *LHS, const SCEV *RHS)
A raw_ostream that writes to an std::string.
static Optional< BinaryOp > MatchBinaryOp(Value *V, DominatorTree &DT)
Try to map V into a BinaryOp, and return None on failure.
const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
void print(raw_ostream &OS) const
Print out the internal representation of this scalar to the specified stream.
const SCEV * getNumIterationsInRange(const ConstantRange &Range, ScalarEvolution &SE) const
Return the number of iterations of this loop that produce values in the specified constant range...
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
succ_range successors(BasicBlock *BB)
const SCEV * getExpr() const override
Returns the SCEV to which this predicate applies, or nullptr if this is a SCEVUnionPredicate.
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
A vector that has set insertion semantics.
void eraseValueFromMap(Value *V)
Erase Value from ValueExprMap and ExprValueMap.
static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, const Loop *L)
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
APInt shl(const APInt &LHS, unsigned shiftAmt)
Left-shift function.
SCEVEqualPredicate(const FoldingSetNodeIDRef ID, const SCEVUnknown *LHS, const SCEVConstant *RHS)
ConstantRange zextOrTrunc(uint32_t BitWidth) const
Make this range have the bit width given by BitWidth.
This file provides utility classes that use RAII to save and restore values.
static cl::opt< unsigned > MaxValueCompareDepth("scalar-evolution-max-value-compare-depth", cl::Hidden, cl::desc("Maximum depth of recursive value complexity comparisons"), cl::init(2))
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
static bool containsConstantSomewhere(const SCEV *StartExpr)
Determine if any of the operands in this SCEV are a constant or if any of the add or multiply express...
This class implements an extremely fast bulk output stream that can only output to a stream...
bool implies(const SCEVPredicate *N) const override
Returns true if this predicate implies N.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
bool isBackedgeTakenCountMaxOrZero(const Loop *L)
Return true if the backedge taken count is either the value returned by getMaxBackedgeTakenCount or z...
const SCEV * getUDivExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
const SCEV * getUnknown(Value *V)
FoldingSetNodeIDRef Intern(BumpPtrAllocator &Allocator) const
Intern - Copy this node's data to a memory region allocated from the given allocator and return a Fol...
The SCEV varies predictably with the loop.
static const SCEV * rewrite(const SCEV *Scev, ScalarEvolution &SE, ValueToValueMap &Map, bool InterpretConsts=false)
The legacy pass manager's analysis pass to compute loop information.
static bool findArrayDimensionsRec(ScalarEvolution &SE, SmallVectorImpl< const SCEV * > &Terms, SmallVectorImpl< const SCEV * > &Sizes)
bool dominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV dominate the specified basic block...
op_iterator op_end() const
Value handle with callbacks on RAUW and destruction.
static int sizeOfSCEV(const SCEV *S)
StringRef - Represent a constant reference to a string, i.e.
inst_range instructions(Function *F)
APInt zext(unsigned width) const
Zero extend to a new width.
A container for analyses that lazily runs them and caches their results.
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
See if we can compute a simplified version of this instruction.
Legacy analysis pass which computes a DominatorTree.
BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB)
Return the "disposition" of the given SCEV with respect to the given block.
const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
bool replacementPreservesLCSSAForm(Instruction *From, Value *To)
Returns true if replacing From with To everywhere is guaranteed to preserve LCSSA form...
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml","ocaml 3.10-compatible collector")
static bool IsMaxConsistingOf(const SCEV *MaybeMaxExpr, const SCEV *Candidate)
Is MaybeMaxExpr an SMax or UMax of Candidate and some other values?
bool isAlwaysTrue() const override
Returns true if the predicate is always true.
const SCEV * getElementSize(Instruction *Inst)
Return the size of an element read or written by Inst.
static bool CanConstantFold(const Instruction *I)
Return true if we can constant fold an instruction of the specified type, assuming that all operands ...
Constant * ConstantFoldLoadThroughGEPIndices(Constant *C, ArrayRef< Constant * > Indices)
ConstantFoldLoadThroughGEPIndices - Given a constant and getelementptr indices (with an implied zero ...
const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty)
bool isLoopInvariantPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L, ICmpInst::Predicate &InvariantPred, const SCEV *&InvariantLHS, const SCEV *&InvariantRHS)
Return true if the result of the predicate LHS Pred RHS is loop invariant with respect to L...
APInt abs() const
Get the absolute value;.
static bool containsUndefs(const SCEV *S)
bool hasComputableLoopEvolution(const SCEV *S, const Loop *L)
Return true if the given SCEV changes value in a known way in the specified loop. ...
This node is a base class providing common functionality for n'ary operators.
static Constant * getMul(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
const SCEV * getNoopOrAnyExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
This class represents an assumption made on an AddRec expression.
PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L)
NoWrapFlags
NoWrapFlags are bitfield indices into SubclassData.
friend class SCEVCallbackVH
static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, const SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
const SCEV * getOperand() const
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
unsigned getLoopDepth() const
Return the nesting level of this loop.
const SCEV * getNotSCEV(const SCEV *V)
Return the SCEV object corresponding to ~V.
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical multiply expression, or something simpler if possible.
bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
Return true if the specified loop has an analyzable loop-invariant backedge-taken count...
This class represents an assumption that two SCEV expressions are equal, and this can be checked at r...
A special type used by analysis passes to provide an address that identifies that particular analysis...
const BasicBlock * getParent() const
static bool classof(const SCEV *S)
Methods for support type inquiry through isa, cast, and dyn_cast:
bool hasNoUnsignedWrap() const
bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv)
This visitor recursively visits a SCEV expression and re-writes it.
bool isQuadratic() const
Return true if this represents an expression A + B*x + C*x^2 where A, B and C are loop invariant valu...
void print(raw_ostream &OS, unsigned Depth=0) const override
Prints a textual representation of this predicate with an indentation of Depth.
This class represents a constant integer value.
const SCEV * getAnyExtendExpr(const SCEV *Op, Type *Ty)
getAnyExtendExpr - Return a SCEV for the given operand extended with unspecified bits out to the give...
APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Is LHS Pred RHS true on the virtue of LHS or RHS being a Min or Max expression?