32 using namespace PatternMatch;
56 if (U->getType() == Ty)
57 if (
CastInst *CI = dyn_cast<CastInst>(U))
58 if (CI->getOpcode() == Op) {
68 CI->replaceAllUsesWith(Ret);
83 assert(SE.DT->dominates(Ret, BIP));
85 rememberInstruction(Ret);
94 assert((Op == Instruction::BitCast ||
95 Op == Instruction::PtrToInt ||
96 Op == Instruction::IntToPtr) &&
97 "InsertNoopCastOfTo cannot perform non-noop casts!");
98 assert(SE.getTypeSizeInBits(V->
getType()) == SE.getTypeSizeInBits(Ty) &&
99 "InsertNoopCastOfTo cannot change sizes!");
102 if (Op == Instruction::BitCast) {
105 if (
CastInst *CI = dyn_cast<CastInst>(V)) {
106 if (CI->getOperand(0)->getType() == Ty)
107 return CI->getOperand(0);
111 if ((Op == Instruction::PtrToInt || Op == Instruction::IntToPtr) &&
112 SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(V->
getType())) {
113 if (
CastInst *CI = dyn_cast<CastInst>(V))
114 if ((CI->getOpcode() == Instruction::PtrToInt ||
115 CI->getOpcode() == Instruction::IntToPtr) &&
116 SE.getTypeSizeInBits(CI->getType()) ==
117 SE.getTypeSizeInBits(CI->getOperand(0)->getType()))
118 return CI->getOperand(0);
120 if ((
CE->getOpcode() == Instruction::PtrToInt ||
121 CE->getOpcode() == Instruction::IntToPtr) &&
122 SE.getTypeSizeInBits(
CE->getType()) ==
123 SE.getTypeSizeInBits(
CE->getOperand(0)->getType()))
124 return CE->getOperand(0);
128 if (
Constant *C = dyn_cast<Constant>(V))
133 if (
Argument *
A = dyn_cast<Argument>(V)) {
135 while ((isa<BitCastInst>(IP) &&
136 isa<Argument>(cast<BitCastInst>(IP)->getOperand(0)) &&
137 cast<BitCastInst>(IP)->getOperand(0) !=
A) ||
138 isa<DbgInfoIntrinsic>(IP) ||
139 isa<LandingPadInst>(IP))
141 return ReuseOrCreateCast(
A, Ty, Op, IP);
148 IP = II->getNormalDest()->begin();
149 while (isa<PHINode>(IP) || isa<LandingPadInst>(IP))
151 return ReuseOrCreateCast(I, Ty, Op, IP);
159 if (
Constant *CLHS = dyn_cast<Constant>(LHS))
160 if (
Constant *CRHS = dyn_cast<Constant>(RHS))
168 if (IP != BlockBegin) {
173 if (isa<DbgInfoIntrinsic>(IP))
175 if (IP->getOpcode() == (
unsigned)Opcode && IP->getOperand(0) == LHS &&
176 IP->getOperand(1) == RHS)
178 if (IP == BlockBegin)
break;
183 DebugLoc Loc = Builder.GetInsertPoint()->getDebugLoc();
184 BuilderType::InsertPointGuard Guard(Builder);
187 while (
const Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock())) {
188 if (!L->isLoopInvariant(LHS) || !L->isLoopInvariant(RHS))
break;
189 BasicBlock *Preheader = L->getLoopPreheader();
190 if (!Preheader)
break;
193 Builder.SetInsertPoint(Preheader, Preheader->
getTerminator());
197 Instruction *BO = cast<Instruction>(Builder.CreateBinOp(Opcode, LHS, RHS));
199 rememberInstruction(BO);
225 if (
const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
233 C->getValue()->getValue().sdiv(
234 FC->getValue()->getValue()));
244 FC->getValue()->getValue())));
252 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
256 if (
const SCEVConstant *C = dyn_cast<SCEVConstant>(M->getOperand(0)))
268 const SCEV *Step =
A->getStepRecurrence(SE);
274 const SCEV *Start =
A->getStart();
292 unsigned NumAddRecs = 0;
293 for (
unsigned i = Ops.
size(); i > 0 && isa<SCEVAddRecExpr>(Ops[i-1]); --i)
299 const SCEV *Sum = NoAddRecs.empty() ?
305 if (
const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Sum))
306 Ops.
append(Add->op_begin(), Add->op_end());
307 else if (!Sum->isZero())
310 Ops.
append(AddRecs.begin(), AddRecs.end());
323 for (
unsigned i = 0, e = Ops.
size(); i != e; ++i)
325 const SCEV *Start =
A->getStart();
326 if (Start->
isZero())
break;
329 A->getStepRecurrence(SE),
332 if (
const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Start)) {
334 Ops.
append(Add->op_begin(), Add->op_end());
335 e += Add->getNumOperands();
340 if (!AddRecs.
empty()) {
375 Value *SCEVExpander::expandAddToGEP(
const SCEV *
const *op_begin,
376 const SCEV *
const *op_end,
381 Type *ElTy = OriginalElTy;
384 bool AnyNonZeroIndices =
false;
390 Type *IntPtrTy =
DL.getIntPtrType(PTy);
402 const SCEV *ElSize = SE.getSizeOfExpr(IntPtrTy, ElTy);
405 for (
unsigned i = 0, e = Ops.size(); i != e; ++i) {
406 const SCEV *Op = Ops[i];
407 const SCEV *Remainder = SE.getConstant(Ty, 0);
413 AnyNonZeroIndices =
true;
421 if (!ScaledOps.
empty()) {
434 expandCodeFor(SE.getAddExpr(ScaledOps), Ty);
438 while (
StructType *STy = dyn_cast<StructType>(ElTy)) {
439 bool FoundFieldNo =
false;
441 if (STy->getNumElements() == 0)
break;
446 if (
const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[0]))
447 if (SE.getTypeSizeInBits(
C->getType()) <= 64) {
449 uint64_t FullOffset =
C->getValue()->getZExtValue();
454 ElTy = STy->getTypeAtIndex(ElIdx);
457 AnyNonZeroIndices =
true;
465 ElTy = STy->getTypeAtIndex(0u);
471 if (
ArrayType *ATy = dyn_cast<ArrayType>(ElTy))
472 ElTy = ATy->getElementType();
480 if (!AnyNonZeroIndices) {
482 V = InsertNoopCastOfTo(V,
485 assert(!isa<Instruction>(V) ||
486 SE.DT->dominates(cast<Instruction>(V), Builder.GetInsertPoint()));
489 Value *Idx = expandCodeFor(SE.getAddExpr(Ops), Ty);
492 if (
Constant *CLHS = dyn_cast<Constant>(V))
493 if (
Constant *CRHS = dyn_cast<Constant>(Idx))
498 unsigned ScanLimit = 6;
502 if (IP != BlockBegin) {
507 if (isa<DbgInfoIntrinsic>(IP))
509 if (IP->getOpcode() == Instruction::GetElementPtr &&
510 IP->getOperand(0) == V && IP->getOperand(1) == Idx)
512 if (IP == BlockBegin)
break;
517 BuilderType::InsertPointGuard Guard(Builder);
520 while (
const Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock())) {
521 if (!L->isLoopInvariant(V) || !L->isLoopInvariant(Idx))
break;
522 BasicBlock *Preheader = L->getLoopPreheader();
523 if (!Preheader)
break;
526 Builder.SetInsertPoint(Preheader, Preheader->
getTerminator());
530 Value *
GEP = Builder.CreateGEP(Builder.getInt8Ty(), V, Idx,
"uglygep");
531 rememberInstruction(GEP);
537 BuilderType::InsertPoint SaveInsertPt = Builder.saveIP();
540 while (
const Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock())) {
541 if (!L->isLoopInvariant(V))
break;
543 bool AnyIndexNotLoopInvariant =
false;
545 E = GepIndices.
end(); I != E; ++
I)
546 if (!L->isLoopInvariant(*I)) {
547 AnyIndexNotLoopInvariant =
true;
550 if (AnyIndexNotLoopInvariant)
553 BasicBlock *Preheader = L->getLoopPreheader();
554 if (!Preheader)
break;
557 Builder.SetInsertPoint(Preheader, Preheader->
getTerminator());
565 Casted = InsertNoopCastOfTo(Casted, PTy);
566 Value *GEP = Builder.CreateGEP(OriginalElTy, Casted,
569 Ops.push_back(SE.getUnknown(GEP));
570 rememberInstruction(GEP);
573 Builder.restoreIP(SaveInsertPt);
575 return expand(SE.getAddExpr(Ops));
594 const Loop *SCEVExpander::getRelevantLoop(
const SCEV *S) {
596 std::pair<DenseMap<const SCEV *, const Loop *>::iterator,
bool> Pair =
597 RelevantLoops.insert(std::make_pair(S,
nullptr));
599 return Pair.first->second;
601 if (isa<SCEVConstant>(S))
604 if (
const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
605 if (
const Instruction *I = dyn_cast<Instruction>(U->getValue()))
606 return Pair.first->second = SE.LI->getLoopFor(I->
getParent());
611 const Loop *L =
nullptr;
617 return RelevantLoops[
N] = L;
619 if (
const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) {
620 const Loop *Result = getRelevantLoop(
C->getOperand());
621 return RelevantLoops[
C] = Result;
623 if (
const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
626 getRelevantLoop(D->getRHS()),
628 return RelevantLoops[D] = Result;
641 bool operator()(std::pair<const Loop *, const SCEV *> LHS,
642 std::pair<const Loop *, const SCEV *> RHS)
const {
644 if (LHS.second->getType()->isPointerTy() !=
645 RHS.second->getType()->isPointerTy())
646 return LHS.second->getType()->isPointerTy();
649 if (LHS.first != RHS.first)
655 if (LHS.second->isNonConstantNegative()) {
656 if (!RHS.second->isNonConstantNegative())
658 }
else if (RHS.second->isNonConstantNegative())
676 for (std::reverse_iterator<SCEVAddExpr::op_iterator>
I(S->
op_end()),
678 OpsAndLoops.
push_back(std::make_pair(getRelevantLoop(*I), *I));
682 std::stable_sort(OpsAndLoops.
begin(), OpsAndLoops.
end(), LoopCompare(*SE.DT));
686 Value *Sum =
nullptr;
688 I = OpsAndLoops.
begin(), E = OpsAndLoops.
end(); I != E; ) {
689 const Loop *CurLoop = I->first;
690 const SCEV *Op = I->second;
695 }
else if (
PointerType *PTy = dyn_cast<PointerType>(Sum->getType())) {
699 for (; I != E && I->first == CurLoop; ++
I) {
702 const SCEV *
X = I->second;
703 if (
const SCEVUnknown *U = dyn_cast<SCEVUnknown>(X))
704 if (!isa<Instruction>(U->getValue()))
705 X = SE.getSCEV(U->getValue());
708 Sum = expandAddToGEP(NewOps.
begin(), NewOps.
end(), PTy, Ty, Sum);
714 NewOps.
push_back(isa<Instruction>(Sum) ? SE.getUnknown(Sum) :
716 for (++I; I != E && I->first == CurLoop; ++
I)
718 Sum = expandAddToGEP(NewOps.
begin(), NewOps.
end(), PTy, Ty, expand(Op));
721 Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty);
722 Sum = InsertNoopCastOfTo(Sum, Ty);
723 Sum = InsertBinop(Instruction::Sub, Sum, W);
727 Value *W = expandCodeFor(Op, Ty);
728 Sum = InsertNoopCastOfTo(Sum, Ty);
730 if (isa<Constant>(Sum))
std::swap(Sum, W);
731 Sum = InsertBinop(Instruction::Add, Sum, W);
745 for (std::reverse_iterator<SCEVMulExpr::op_iterator>
I(S->
op_end()),
747 OpsAndLoops.
push_back(std::make_pair(getRelevantLoop(*I), *I));
750 std::stable_sort(OpsAndLoops.
begin(), OpsAndLoops.
end(), LoopCompare(*SE.DT));
754 Value *Prod =
nullptr;
756 I = OpsAndLoops.
begin(), E = OpsAndLoops.
end(); I != E; ++
I) {
757 const SCEV *Op = I->second;
763 Prod = InsertNoopCastOfTo(Prod, Ty);
767 Value *W = expandCodeFor(Op, Ty);
768 Prod = InsertNoopCastOfTo(Prod, Ty);
770 if (isa<Constant>(Prod))
std::swap(Prod, W);
774 assert(!Ty->
isVectorTy() &&
"vector types are not SCEVable");
775 Prod = InsertBinop(Instruction::Shl, Prod,
778 Prod = InsertBinop(Instruction::Mul, Prod, W);
791 const APInt &RHS =
SC->getValue()->getValue();
793 return InsertBinop(Instruction::LShr, LHS,
798 return InsertBinop(Instruction::UDiv, LHS, RHS);
807 Base =
A->getStart();
810 A->getStepRecurrence(SE),
814 if (
const SCEVAddExpr *
A = dyn_cast<SCEVAddExpr>(Base)) {
815 Base =
A->getOperand(
A->getNumOperands()-1);
817 NewAddOps.
back() = Rest;
828 (isa<CastInst>(IncV) && !isa<BitCastInst>(IncV)))
833 if (L == IVIncInsertLoop) {
835 OE = IncV->
op_end(); OI != OE; ++OI)
836 if (
Instruction *OInst = dyn_cast<Instruction>(OI))
837 if (!SE.DT->dominates(OInst, IVIncInsertPos))
851 return isNormalAddRecExprPHI(PN, IncV, L);
866 if (IncV == InsertPos)
873 case Instruction::Add:
874 case Instruction::Sub: {
876 if (!OInst || SE.DT->dominates(OInst, InsertPos))
877 return dyn_cast<Instruction>(IncV->
getOperand(0));
880 case Instruction::BitCast:
882 case Instruction::GetElementPtr:
885 if (isa<Constant>(*I))
887 if (
Instruction *OInst = dyn_cast<Instruction>(*I)) {
888 if (!SE.DT->dominates(OInst, InsertPos))
901 unsigned AS = cast<PointerType>(IncV->
getType())->getAddressSpace();
915 if (SE.DT->dominates(IncV, InsertPos))
920 if (isa<PHINode>(InsertPos)
927 Instruction *Oper = getIVIncOperand(IncV, InsertPos,
true);
933 if (SE.DT->dominates(IncV, InsertPos))
937 E = IVIncs.
rend(); I != E; ++
I) {
968 PointerType *GEPPtrTy = cast<PointerType>(ExpandTy);
971 if (!isa<ConstantInt>(StepV))
974 const SCEV *
const StepArray[1] = { SE.getSCEV(StepV) };
975 IncV = expandAddToGEP(StepArray, StepArray+1, GEPPtrTy, IntTy, PN);
977 IncV = Builder.CreateBitCast(IncV, PN->
getType());
978 rememberInstruction(IncV);
982 Builder.CreateSub(PN, StepV,
Twine(IVName) +
".iv.next") :
983 Builder.CreateAdd(PN, StepV,
Twine(IVName) +
".iv.next");
984 rememberInstruction(IncV);
1000 InstToHoist = cast<Instruction>(InstToHoist->
getOperand(0));
1001 }
while (InstToHoist != LoopPhi);
1022 if (Phi == Requested) {
1038 if (!isa<IntegerType>(AR->
getType()))
1046 const SCEV *ExtendAfterOp =
1048 return ExtendAfterOp == OpAfterExtend;
1052 if (!isa<IntegerType>(AR->
getType()))
1060 const SCEV *ExtendAfterOp =
1062 return ExtendAfterOp == OpAfterExtend;
1069 SCEVExpander::getAddRecExprPHILiterally(
const SCEVAddRecExpr *Normalized,
1075 assert((!IVIncInsertLoop||IVIncInsertPos) &&
"Uninitialized insert position");
1080 PHINode *AddRecPhiMatch =
nullptr;
1087 bool TryNonMatchingSCEV = IVIncInsertLoop &&
1088 SE.DT->properlyDominates(LatchBlock, IVIncInsertLoop->getHeader());
1092 if (!SE.isSCEVable(PN->
getType()))
1099 bool IsMatchingSCEV = PhiSCEV == Normalized;
1103 if (!IsMatchingSCEV && !TryNonMatchingSCEV)
1111 if (!isExpandedAddRecExprPHI(PN, TempIncV, L))
1113 if (L == IVIncInsertLoop && !hoistIVInc(TempIncV, IVIncInsertPos))
1116 if (!isNormalAddRecExprPHI(PN, TempIncV, L))
1121 if (IsMatchingSCEV) {
1125 AddRecPhiMatch = PN;
1131 if ((!TruncTy || InvertStep) &&
1135 AddRecPhiMatch = PN;
1137 TruncTy = SE.getEffectiveSCEVType(Normalized->
getType());
1141 if (AddRecPhiMatch) {
1144 if (L == IVIncInsertLoop)
1149 InsertedValues.insert(AddRecPhiMatch);
1151 rememberInstruction(IncV);
1152 return AddRecPhiMatch;
1157 BuilderType::InsertPointGuard Guard(Builder);
1167 PostIncLoops.
clear();
1170 Value *StartV = expandCodeFor(Normalized->
getStart(), ExpandTy,
1174 assert(!isa<Instruction>(StartV) ||
1175 SE.DT->properlyDominates(cast<Instruction>(StartV)->getParent(),
1186 Step = SE.getNegativeSCEV(Step);
1188 Value *StepV = expandCodeFor(Step, IntTy, L->
getHeader()->begin());
1193 bool IncrementIsNUW = !useSubtract &&
IsIncrementNUW(SE, Normalized);
1194 bool IncrementIsNSW = !useSubtract &&
IsIncrementNSW(SE, Normalized);
1198 Builder.SetInsertPoint(Header, Header->
begin());
1200 PHINode *PN = Builder.CreatePHI(ExpandTy, std::distance(HPB, HPE),
1201 Twine(IVName) +
".iv");
1202 rememberInstruction(PN);
1219 Builder.SetInsertPoint(InsertPos);
1220 Value *IncV = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
1222 if (isa<OverflowingBinaryOperator>(IncV)) {
1224 cast<BinaryOperator>(IncV)->setHasNoUnsignedWrap();
1226 cast<BinaryOperator>(IncV)->setHasNoSignedWrap();
1233 PostIncLoops = SavedPostIncLoops;
1236 InsertedValues.
insert(PN);
1243 Type *IntTy = SE.getEffectiveSCEVType(STy);
1249 if (PostIncLoops.count(L)) {
1254 nullptr, Loops, SE, *SE.DT));
1259 const SCEV *PostLoopOffset =
nullptr;
1260 if (!SE.properlyDominates(Start, L->
getHeader())) {
1261 PostLoopOffset = Start;
1262 Start = SE.getConstant(Normalized->
getType(), 0);
1263 Normalized = cast<SCEVAddRecExpr>(
1271 const SCEV *PostLoopScale =
nullptr;
1272 if (!SE.dominates(Step, L->
getHeader())) {
1273 PostLoopScale = Step;
1274 Step = SE.getConstant(Normalized->
getType(), 1);
1276 cast<SCEVAddRecExpr>(SE.getAddRecExpr(
1277 Start, Step, Normalized->
getLoop(),
1283 Type *ExpandTy = PostLoopScale ? IntTy : STy;
1286 Type *TruncTy =
nullptr;
1287 bool InvertStep =
false;
1288 PHINode *PN = getAddRecExprPHILiterally(Normalized, L, ExpandTy, IntTy,
1289 TruncTy, InvertStep);
1293 if (!PostIncLoops.count(L))
1298 assert(LatchBlock &&
"PostInc mode requires a unique loop latch!");
1304 if (isa<Instruction>(Result)
1305 && !SE.DT->dominates(cast<Instruction>(Result),
1306 Builder.GetInsertPoint())) {
1319 Step = SE.getNegativeSCEV(Step);
1323 BuilderType::InsertPointGuard Guard(Builder);
1324 StepV = expandCodeFor(Step, IntTy, L->
getHeader()->begin());
1326 Result = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
1335 if (ResTy != SE.getEffectiveSCEVType(ResTy))
1336 Result = InsertNoopCastOfTo(Result, SE.getEffectiveSCEVType(ResTy));
1338 if (TruncTy != Result->
getType()) {
1339 Result = Builder.CreateTrunc(Result, TruncTy);
1340 rememberInstruction(Result);
1344 Result = Builder.CreateSub(expandCodeFor(Normalized->
getStart(), TruncTy),
1346 rememberInstruction(Result);
1351 if (PostLoopScale) {
1352 assert(S->
isAffine() &&
"Can't linearly scale non-affine recurrences.");
1353 Result = InsertNoopCastOfTo(Result, IntTy);
1354 Result = Builder.CreateMul(Result,
1355 expandCodeFor(PostLoopScale, IntTy));
1356 rememberInstruction(Result);
1360 if (PostLoopOffset) {
1361 if (
PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
1362 const SCEV *
const OffsetArray[1] = { PostLoopOffset };
1363 Result = expandAddToGEP(OffsetArray, OffsetArray+1, PTy, IntTy, Result);
1365 Result = InsertNoopCastOfTo(Result, IntTy);
1366 Result = Builder.CreateAdd(Result,
1367 expandCodeFor(PostLoopOffset, IntTy));
1368 rememberInstruction(Result);
1376 if (!CanonicalMode)
return expandAddRecExprLiterally(S);
1382 PHINode *CanonicalIV =
nullptr;
1384 if (SE.getTypeSizeInBits(PN->
getType()) >= SE.getTypeSizeInBits(Ty))
1390 SE.getTypeSizeInBits(CanonicalIV->
getType()) >
1391 SE.getTypeSizeInBits(Ty)) {
1394 NewOps[i] = SE.getAnyExtendExpr(S->
op_begin()[i], CanonicalIV->
getType());
1395 Value *V = expand(SE.getAddRecExpr(NewOps, S->
getLoop(),
1399 BuilderType::InsertPointGuard Guard(Builder);
1400 while (isa<PHINode>(NewInsertPt) || isa<DbgInfoIntrinsic>(NewInsertPt) ||
1401 isa<LandingPadInst>(NewInsertPt))
1403 V = expandCodeFor(SE.getTruncateExpr(SE.getUnknown(V), Ty),
nullptr,
1411 NewOps[0] = SE.getConstant(Ty, 0);
1412 const SCEV *Rest = SE.getAddRecExpr(NewOps, L,
1418 const SCEV *RestArray[1] = { Rest };
1426 if (!isa<SCEVMulExpr>(Base) && !isa<SCEVUDivExpr>(Base)) {
1427 Value *StartV = expand(Base);
1428 assert(StartV->
getType() == PTy &&
"Pointer type mismatch for GEP!");
1429 return expandAddToGEP(RestArray, RestArray+1, PTy, Ty, StartV);
1434 return expand(SE.getAddExpr(SE.getUnknown(expand(S->
getStart())),
1435 SE.getUnknown(expand(Rest))));
1446 rememberInstruction(CanonicalIV);
1452 if (!PredSeen.
insert(HP).second) {
1466 rememberInstruction(Add);
1476 assert(Ty == SE.getEffectiveSCEVType(CanonicalIV->
getType()) &&
1477 "IVs with types different from the canonical IV should "
1478 "already have been handled!");
1487 expand(SE.getTruncateOrNoop(
1488 SE.getMulExpr(SE.getUnknown(CanonicalIV),
1497 const SCEV *IH = SE.getUnknown(CanonicalIV);
1500 const SCEV *NewS = S;
1501 const SCEV *
Ext = SE.getNoopOrAnyExtend(S, CanonicalIV->
getType());
1502 if (isa<SCEVAddRecExpr>(Ext))
1505 const SCEV *V = cast<SCEVAddRecExpr>(NewS)->evaluateAtIteration(IH, SE);
1509 const SCEV *
T = SE.getTruncateOrNoop(V, Ty);
1517 Value *I = Builder.CreateTrunc(V, Ty);
1518 rememberInstruction(I);
1526 Value *I = Builder.CreateZExt(V, Ty);
1527 rememberInstruction(I);
1535 Value *I = Builder.CreateSExt(V, Ty);
1536 rememberInstruction(I);
1547 Ty = SE.getEffectiveSCEVType(Ty);
1548 LHS = InsertNoopCastOfTo(LHS, Ty);
1551 Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
1552 rememberInstruction(ICmp);
1553 Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS,
"smax");
1554 rememberInstruction(Sel);
1560 LHS = InsertNoopCastOfTo(LHS, S->
getType());
1571 Ty = SE.getEffectiveSCEVType(Ty);
1572 LHS = InsertNoopCastOfTo(LHS, Ty);
1575 Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
1576 rememberInstruction(ICmp);
1577 Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS,
"umax");
1578 rememberInstruction(Sel);
1584 LHS = InsertNoopCastOfTo(LHS, S->
getType());
1590 Builder.SetInsertPoint(IP->
getParent(), IP);
1591 return expandCodeFor(SH, Ty);
1596 Value *V = expand(SH);
1598 assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->
getType()) &&
1599 "non-trivial casts should be done with the SCEVs directly!");
1600 V = InsertNoopCastOfTo(V, Ty);
1605 Value *SCEVExpander::expand(
const SCEV *S) {
1609 for (
Loop *L = SE.LI->getLoopFor(Builder.GetInsertBlock()); ;
1611 if (SE.isLoopInvariant(S, L)) {
1619 InsertPt = L->
getHeader()->getFirstInsertionPt();
1625 if (L && SE.hasComputableLoopEvolution(S, L) && !PostIncLoops.count(L))
1626 InsertPt = L->
getHeader()->getFirstInsertionPt();
1627 while (InsertPt != Builder.GetInsertPoint()
1628 && (isInsertedInstruction(InsertPt)
1629 || isa<DbgInfoIntrinsic>(InsertPt))) {
1637 I = InsertedExpressions.find(std::make_pair(S, InsertPt));
1638 if (I != InsertedExpressions.end())
1641 BuilderType::InsertPointGuard Guard(Builder);
1642 Builder.SetInsertPoint(InsertPt->
getParent(), InsertPt);
1645 Value *V = visit(S);
1653 InsertedExpressions[std::make_pair(S, InsertPt)] = V;
1657 void SCEVExpander::rememberInstruction(
Value *I) {
1658 if (!PostIncLoops.empty())
1659 InsertedPostIncValues.insert(I);
1661 InsertedValues.insert(I);
1671 assert(Ty->
isIntegerTy() &&
"Can only insert integer induction variables!");
1675 const SCEV *
H = SE.getAddRecExpr(SE.getConstant(Ty, 0),
1680 PHINode *V = cast<PHINode>(expandCodeFor(H,
nullptr,
1710 unsigned NumElim = 0;
1715 PEnd = Phis.
end(); PIter != PEnd; ++PIter) {
1725 <<
"INDVARS: Eliminated constant iv: " << *Phi <<
'\n');
1729 if (!SE.isSCEVable(Phi->
getType()))
1732 PHINode *&OrigPhiRef = ExprToIVMap[SE.getSCEV(Phi)];
1739 const SCEV *TruncExpr =
1740 SE.getTruncateExpr(SE.getSCEV(Phi), Phis.
back()->
getType());
1741 ExprToIVMap[TruncExpr] = Phi;
1761 && !(ChainedPhis.count(Phi)
1762 || isExpandedAddRecExprPHI(OrigPhiRef, OrigInc, L))
1763 && (ChainedPhis.count(Phi)
1764 || isExpandedAddRecExprPHI(Phi, IsomorphicInc, L))) {
1774 const SCEV *TruncExpr = SE.getTruncateOrNoop(SE.getSCEV(OrigInc),
1775 IsomorphicInc->getType());
1776 if (OrigInc != IsomorphicInc
1777 && TruncExpr == SE.getSCEV(IsomorphicInc)
1778 && ((isa<PHINode>(OrigInc) && isa<PHINode>(IsomorphicInc))
1779 || hoistIVInc(OrigInc, IsomorphicInc))) {
1781 <<
"INDVARS: Eliminated congruent iv.inc: "
1782 << *IsomorphicInc <<
'\n');
1783 Value *NewInc = OrigInc;
1784 if (OrigInc->
getType() != IsomorphicInc->getType()) {
1786 if (
PHINode *PN = dyn_cast<PHINode>(OrigInc))
1794 CreateTruncOrBitCast(OrigInc, IsomorphicInc->
getType(), IVName);
1796 IsomorphicInc->replaceAllUsesWith(NewInc);
1801 <<
"INDVARS: Eliminated congruent iv: " << *Phi <<
'\n');
1803 Value *NewIV = OrigPhiRef;
1807 NewIV = Builder.CreateTruncOrBitCast(OrigPhiRef, Phi->
getType(), IVName);
1815 bool SCEVExpander::isHighCostExpansionHelper(
1824 return isHighCostExpansionHelper(cast<SCEVTruncateExpr>(S)->getOperand(), L,
1827 return isHighCostExpansionHelper(cast<SCEVZeroExtendExpr>(S)->getOperand(),
1830 return isHighCostExpansionHelper(cast<SCEVSignExtendExpr>(S)->getOperand(),
1834 if (!Processed.
insert(S).second)
1837 if (
auto *UDivExpr = dyn_cast<SCEVUDivExpr>(S)) {
1841 if (
auto *
SC = dyn_cast<SCEVConstant>(UDivExpr->getRHS()))
1842 if (
SC->getValue()->getValue().isPowerOf2()) {
1844 L->
getHeader()->getParent()->getParent()->getDataLayout();
1845 unsigned Width = cast<IntegerType>(UDivExpr->getType())->
getBitWidth();
1867 RHS = SE.getMinusSCEV(RHS, SE.getConstant(RHS->
getType(), 1));
1870 LHS = SE.getMinusSCEV(LHS, SE.getConstant(LHS->
getType(), 1));
1878 if (isa<SCEVSMaxExpr>(S) || isa<SCEVUMaxExpr>(S))
1884 if (
const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(S)) {
1887 if (isHighCostExpansionHelper(*I, L, Processed))
1918 struct SCEVFindUnsafe {
1924 bool follow(
const SCEV *S) {
1925 if (
const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
1933 const SCEV *Step = AR->getStepRecurrence(SE);
1934 if (!AR->isAffine() && !SE.dominates(Step, AR->getLoop()->getHeader())) {
1941 bool isDone()
const {
return IsUnsafe; }
1947 SCEVFindUnsafe Search(SE);
1949 return !Search.IsUnsafe;
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask=NoWrapMask) const
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
const SCEV * getTruncateOrNoop(const SCEV *V, Type *Ty)
getTruncateOrNoop - Return a SCEV corresponding to a conversion of the input value to the specified t...
std::reverse_iterator< iterator > reverse_iterator
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
bool hoistIVInc(Instruction *IncV, Instruction *InsertPos)
Utility for hoisting an IV increment.
static IntegerType * getInt1Ty(LLVMContext &C)
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
LLVM Argument representation.
const SCEV * TransformForPostIncUse(TransformKind Kind, const SCEV *S, Instruction *User, Value *OperandValToReplace, PostIncLoopSet &Loops, ScalarEvolution &SE, DominatorTree &DT)
TransformForPostIncUse - Transform the given expression according to the given transformation kind...
bool isOne() const
isOne - Return true if the expression is a constant one.
const SCEV * getConstant(ConstantInt *V)
LLVMContext & getContext() const
bool isZero() const
isZero - Return true if the expression is a constant zero.
unsigned getNumOperands() const
ScalarEvolution - This class is the main scalar evolution driver.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
PointerType::get - This constructs a pointer to an object of the specified type in a numbered address...
bool mayHaveSideEffects() const
mayHaveSideEffects - Return true if the instruction may have side effects.
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
bool isIllegalInteger(unsigned Width) const
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
getStepRecurrence - This method constructs and returns the recurrence indicating how much this expres...
SCEVTruncateExpr - This class represents a truncation of an integer value to a smaller integer value...
LoopT * getParentLoop() const
unsigned getAddressSpace() const
Return the address space of the Pointer type.
BlockT * getHeader() const
SCEVCastExpr - This is the base class for unary cast operator classes.
const SCEV * getStart() const
static Constant * getNullValue(Type *Ty)
StringRef getName() const
Return a constant reference to the value's name.
BlockT * getLoopLatch() const
getLoopLatch - If there is a single latch block for this loop, return it.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
iterator begin()
Instruction iterator methods.
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
bool match(Val *V, const Pattern &P)
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static const Loop * PickMostRelevantLoop(const Loop *A, const Loop *B, DominatorTree &DT)
PickMostRelevantLoop - Given two loops pick the one that's most relevant for SCEV expansion...
This is the base class for all instructions that perform data casts.
NodeTy * getNextNode()
Get the next node, or 0 for the list tail.
const APInt & getValue() const
Return the constant as an APInt value reference.
StructType - Class to represent struct types.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
A Use represents the edge between a Value definition and its users.
const SCEV *const * op_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
bool isSized(SmallPtrSetImpl< const Type * > *Visited=nullptr) const
isSized - Return true if it makes sense to take the size of this type.
Windows NT (Windows on ARM)
op_iterator op_begin() const
SCEVMulExpr - This node represents multiplication of some number of SCEVs.
static Constant * get(unsigned Opcode, Constant *C1, Constant *C2, unsigned Flags=0, Type *OnlyIfReducedTy=nullptr)
get - Return a binary or shift operator constant expression, folding if possible. ...
ConstantExpr - a constant value that is initialized with an expression using other constant values...
LLVMContext & getContext() const
getContext - Return the LLVMContext in which this type was uniqued.
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
ArrayType - Class to represent array types.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power of 2.
Type * getEffectiveSCEVType(Type *Ty) const
getEffectiveSCEVType - Return a type with the same bitwidth as the given type and which represents ho...
void takeName(Value *V)
Transfer the name from V to this value.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Type * getElementType() const
const SCEV * getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, SCEV::NoWrapFlags Flags)
getAddRecExpr - Get an add recurrence expression for the specified loop.
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, Instruction *InsertBefore, Value *FlagsOp)
size_t getNumOperands() const
PointerType - Class to represent pointers.
uint64_t getElementOffset(unsigned Idx) const
SCEVUnknown - This means that we are dealing with an entirely unknown SCEV value, and only represent ...
bool isAffine() const
isAffine - Return true if this represents an expression A + B*x where A and B are loop invariant valu...
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
SCEVUDivExpr - This class represents a binary unsigned division operation.
The instances of the Type class are immutable: once they are created, they are never changed...
Type * getType() const
getType - Return the LLVM type of this SCEV expression.
BranchInst - Conditional or Unconditional Branch instruction.
static void SplitAddRecs(SmallVectorImpl< const SCEV * > &Ops, Type *Ty, ScalarEvolution &SE)
SplitAddRecs - Flatten a list of add operands, moving addrec start values out to the top level...
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Value handle that tracks a Value across RAUW.
This is an important base class in LLVM.
const SCEV * getOperand(unsigned i) const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Normalize - Normalize according to the given loops.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
bool contains(const LoopT *L) const
contains - Return true if the specified loop is contained within in this loop.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
This instruction compares its operands according to the predicate given to the constructor.
static bool FactorOutConstant(const SCEV *&S, const SCEV *&Remainder, const SCEV *Factor, ScalarEvolution &SE, const DataLayout &DL)
FactorOutConstant - Test if S is divisible by Factor, using signed division.
Value * expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I)
Insert code to directly compute the specified SCEV expression into the program.
BlockT * getExitingBlock() const
getExitingBlock - If getExitingBlocks would return exactly one block, return that block...
Value * getOperand(unsigned i) const
Interval::pred_iterator pred_end(Interval *I)
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
const SCEV * getLHS() const
static void SimplifyAddOperands(SmallVectorImpl< const SCEV * > &Ops, Type *Ty, ScalarEvolution &SE)
SimplifyAddOperands - Sort and simplify a list of add operands.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
bool isNonConstantNegative() const
isNonConstantNegative - Return true if the specified scev is negated, but not a constant.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
const SCEV * getRHS() const
static bool IsIncrementNSW(ScalarEvolution &SE, const SCEVAddRecExpr *AR)
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
static void ExposePointerBase(const SCEV *&Base, const SCEV *&Rest, ScalarEvolution &SE)
Move parts of Base into Rest to leave Base with the minimal expression that provides a pointer operan...
bool isConditional() const
unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT, SmallVectorImpl< WeakVH > &DeadInsts, const TargetTransformInfo *TTI=nullptr)
replace congruent phis with their most canonical representative.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
static PointerType * getInt1PtrTy(LLVMContext &C, unsigned AS=0)
unsigned getIntegerBitWidth() const
This is the shared class of boolean and integer constants.
unsigned logBase2() const
PHINode * getOrInsertCanonicalInductionVariable(const Loop *L, Type *Ty)
This method returns the canonical induction variable of the specified type for the specified loop (in...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
bool isAllOnesValue() const
isAllOnesValue - Return true if the expression is a constant all-ones value.
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
CHAIN = SC CHAIN, Imm128 - System call.
uint64_t getSizeInBytes() const
PHINode * getCanonicalInductionVariable() const
getCanonicalInductionVariable - Check to see if the loop has a canonical induction variable: an integ...
unsigned getElementContainingOffset(uint64_t Offset) const
Given a valid byte offset into the structure, returns the structure index that contains it...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
ConstantInt * getValue() const
static void hoistBeforePos(DominatorTree *DT, Instruction *InstToHoist, Instruction *Pos, PHINode *LoopPhi)
Hoist the addrec instruction chain rooted in the loop phi above the position.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Class for arbitrary precision integers.
SCEVAddExpr - This node represents an addition of some number of SCEVs.
const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty)
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
SCEVSMaxExpr - This class represents a signed maximum selection.
iterator_range< user_iterator > users()
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static Constant * getCast(unsigned ops, Constant *C, Type *Ty, bool OnlyIfReduced=false)
Convenience function for getting a Cast operation.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getAddExpr - Get a canonical add expression, or something simpler if possible.
void visitAll(const SCEV *Root, SV &Visitor)
Use SCEVTraversal to visit all nodes in the given expression tree.
SCEVZeroExtendExpr - This class represents a zero extension of a small integer value to a larger inte...
static bool IsIncrementNUW(ScalarEvolution &SE, const SCEVAddRecExpr *AR)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Value * getCondition() const
void emplace_back(ArgTypes &&...Args)
SCEV - This class represents an analyzed expression in the program.
static IntegerType * getInt32Ty(LLVMContext &C)
static Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static cl::opt< unsigned > ScanLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
SCEVSignExtendExpr - This class represents a sign extension of a small integer value to a larger inte...
SCEVUMaxExpr - This class represents an unsigned maximum selection.
Instruction * getIVIncOperand(Instruction *IncV, Instruction *InsertPos, bool allowScale)
Return the induction variable increment's IV operand.
const Loop * getLoop() const
reverse_iterator rbegin()
unsigned getSCEVType() const
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
getPrimitiveSizeInBits - Return the basic size of this type if it is a primitive type.
LLVM Value Representation.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
void moveBefore(Instruction *MovePos)
moveBefore - Unlink this instruction from its current basic block and insert it into the basic block ...
InvokeInst - Invoke instruction.
C - The default llvm calling convention, compatible with C.
op_iterator op_end() const
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
SimplifyInstruction - See if we can compute a simplified version of this instruction.
const SCEV * getNegativeSCEV(const SCEV *V)
getNegativeSCEV - Return the SCEV object corresponding to -V.
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty)
SCEVNAryExpr - This node is a base class providing common functionality for n'ary operators...
bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE)
Return true if the given expression is safe to expand in the sense that all materialized values are s...
const SCEV * getOperand() const
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getMulExpr - Get a canonical multiply expression, or something simpler if possible.
static IntegerType * getInt8Ty(LLVMContext &C)
const BasicBlock * getParent() const
static bool canBeCheaplyTransformed(ScalarEvolution &SE, const SCEVAddRecExpr *Phi, const SCEVAddRecExpr *Requested, bool &InvertStep)
Check whether we can cheaply express the requested SCEV in terms of the available PHI SCEV by truncat...
SCEVConstant - This class represents a constant integer value.