40 #define DEBUG_TYPE "loop-reroll"
42 STATISTIC(NumRerolledLoops,
"Number of rerolled loops");
46 cl::desc(
"The maximum increment for loop rerolling"));
51 cl::desc(
"The maximum number of failures to tolerate"
52 " during fuzzy matching. (default: 400)"));
133 IL_MaxRerollIterations = 16,
140 class LoopReroll :
public LoopPass {
172 struct SimpleLoopReduction {
174 : Valid(
false), Instructions(1, P) {
175 assert(isa<PHINode>(P) &&
"First reduction instruction must be a PHI");
184 assert(Valid &&
"Using invalid reduction");
185 return Instructions.front();
189 assert(Valid &&
"Using invalid reduction");
190 return Instructions.back();
194 assert(Valid &&
"Using invalid reduction");
195 return Instructions[i+1];
198 Instruction *operator [] (
size_t i)
const {
return get(i); }
201 size_t size()
const {
202 assert(Valid &&
"Using invalid reduction");
203 return Instructions.size()-1;
206 typedef SmallInstructionVector::iterator iterator;
207 typedef SmallInstructionVector::const_iterator const_iterator;
210 assert(Valid &&
"Using invalid reduction");
211 return std::next(Instructions.begin());
214 const_iterator
begin()
const {
215 assert(Valid &&
"Using invalid reduction");
216 return std::next(Instructions.begin());
219 iterator
end() {
return Instructions.
end(); }
220 const_iterator
end()
const {
return Instructions.
end(); }
224 SmallInstructionVector Instructions;
231 struct ReductionTracker {
235 void addSLR(SimpleLoopReduction &SLR) { PossibleReds.
push_back(SLR); }
245 void restrictToScale(uint64_t Scale,
246 SmallInstructionSet &PossibleRedSet,
247 SmallInstructionSet &PossibleRedPHISet,
248 SmallInstructionSet &PossibleRedLastSet) {
249 PossibleRedIdx.clear();
250 PossibleRedIter.clear();
253 for (
unsigned i = 0, e = PossibleReds.size(); i != e; ++i)
254 if (PossibleReds[i].
size() % Scale == 0) {
255 PossibleRedLastSet.insert(PossibleReds[i].getReducedValue());
256 PossibleRedPHISet.insert(PossibleReds[i].getPHI());
258 PossibleRedSet.insert(PossibleReds[i].getPHI());
259 PossibleRedIdx[PossibleReds[i].getPHI()] = i;
261 PossibleRedSet.insert(J);
262 PossibleRedIdx[J] = i;
273 if (J1I != PossibleRedIdx.
end()) {
275 if (J2I != PossibleRedIdx.
end() && J1I->second == J2I->second)
286 if (PossibleRedIdx.count(J1)) {
287 assert(PossibleRedIdx.count(J2) &&
288 "Recording reduction vs. non-reduction instruction?");
290 PossibleRedIter[J1] = 0;
291 PossibleRedIter[J2] = i;
293 int Idx = PossibleRedIdx[J1];
294 assert(Idx == PossibleRedIdx[J2] &&
295 "Recording pair from different reductions?");
306 if (!isa<PHINode>(J))
312 if (cast<Instruction>(J) == PossibleReds[i].getPHI())
319 bool validateSelected();
320 void replaceSelected();
324 SmallReductionVector PossibleReds;
359 SmallInstructionVector Roots;
361 SmallInstructionSet SubsumedInsts;
366 struct DAGRootTracker {
370 : Parent(Parent), L(L), SE(SE), AA(AA), TLI(TLI), IV(IV) {}
375 bool validate(ReductionTracker &Reductions);
379 void replace(
const SCEV *IterCount);
385 SmallInstructionSet SubsumedInsts);
386 bool findRootsBase(
Instruction *IVU, SmallInstructionSet SubsumedInsts);
388 std::map<int64_t,Instruction*> &Roots);
390 bool collectUsedInstructions(SmallInstructionSet &PossibleRedSet);
391 void collectInLoopUserSet(
const SmallInstructionVector &Roots,
392 const SmallInstructionSet &Exclude,
393 const SmallInstructionSet &Final,
396 const SmallInstructionSet &Exclude,
397 const SmallInstructionSet &Final,
400 UsesTy::iterator nextInstr(
int Val, UsesTy &
In,
401 const SmallInstructionSet &Exclude,
402 UsesTy::iterator *StartI=
nullptr);
406 UsesTy::iterator Start,
407 UsesTy::iterator End);
428 SmallInstructionVector LoopIncs;
435 void collectPossibleIVs(
Loop *L, SmallInstructionVector &PossibleIVs);
436 void collectPossibleReductions(
Loop *L,
437 ReductionTracker &Reductions);
439 ReductionTracker &Reductions);
453 return new LoopReroll;
461 if (!L->
contains(cast<Instruction>(U)))
469 void LoopReroll::collectPossibleIVs(
Loop *L,
470 SmallInstructionVector &PossibleIVs) {
474 if (!isa<PHINode>(
I))
476 if (!
I->getType()->isIntegerTy())
480 dyn_cast<SCEVAddRecExpr>(SE->getSCEV(
I))) {
481 if (PHISCEV->getLoop() != L)
483 if (!PHISCEV->isAffine())
486 dyn_cast<SCEVConstant>(PHISCEV->getStepRecurrence(*SE))) {
487 if (!IncSCEV->getValue()->getValue().isStrictlyPositive())
489 if (IncSCEV->getValue()->uge(
MaxInc))
492 DEBUG(
dbgs() <<
"LRR: Possible IV: " << *
I <<
" = " <<
494 PossibleIVs.push_back(
I);
504 assert(!Valid &&
"Cannot add to an already-valid chain");
519 if (!(isa<PHINode>(Instructions.back()) ||
523 Instructions.push_back(C);
527 if (Instructions.size() < 2 ||
535 if (L->
contains(cast<Instruction>(U)))
536 if (cast<Instruction>(U) != Instructions.front())
540 Instructions.push_back(C);
545 void LoopReroll::collectPossibleReductions(
Loop *L,
546 ReductionTracker &Reductions) {
550 if (!isa<PHINode>(
I))
552 if (!
I->getType()->isSingleValueType())
555 SimpleLoopReduction SLR(
I, L);
559 DEBUG(
dbgs() <<
"LRR: Possible reduction: " << *
I <<
" (with " <<
560 SLR.size() <<
" chained instructions)\n");
561 Reductions.addSLR(SLR);
577 void LoopReroll::DAGRootTracker::collectInLoopUserSet(
578 Instruction *Root,
const SmallInstructionSet &Exclude,
579 const SmallInstructionSet &Final,
581 SmallInstructionVector Queue(1, Root);
582 while (!Queue.empty()) {
584 if (!Users.
insert(I).second)
590 if (
PHINode *PN = dyn_cast<PHINode>(User)) {
592 if (PN->getIncomingBlock(U) == L->
getHeader())
596 if (L->
contains(User) && !Exclude.count(User)) {
597 Queue.push_back(User);
603 OIE = I->
op_end(); OI != OIE; ++OI) {
605 if (Op->hasOneUse() && L->
contains(Op) && !Exclude.count(Op) &&
614 void LoopReroll::DAGRootTracker::collectInLoopUserSet(
615 const SmallInstructionVector &Roots,
616 const SmallInstructionSet &Exclude,
617 const SmallInstructionSet &Final,
619 for (SmallInstructionVector::const_iterator I = Roots.begin(),
620 IE = Roots.end(); I !=
IE; ++
I)
621 collectInLoopUserSet(*I, Exclude, Final, Users);
625 if (
LoadInst *LI = dyn_cast<LoadInst>(I))
626 return LI->isSimple();
628 return SI->isSimple();
630 return !
MI->isVolatile();
640 default:
return false;
641 case Instruction::Add:
642 case Instruction::Sub:
643 case Instruction::Mul:
644 case Instruction::Shl:
645 case Instruction::AShr:
646 case Instruction::LShr:
647 case Instruction::GetElementPtr:
648 case Instruction::Trunc:
649 case Instruction::ZExt:
650 case Instruction::SExt:
659 if (!BO || BO->
getOpcode() != Instruction::Add)
662 for (
auto *UU : BO->
users()) {
670 bool LoopReroll::DAGRootTracker::
671 collectPossibleRoots(
Instruction *Base, std::map<int64_t,Instruction*> &Roots) {
672 SmallInstructionVector BaseUsers;
674 for (
auto *I : Base->
users()) {
678 LoopIncs.push_back(cast<Instruction>(I));
683 if (
auto *BO = dyn_cast<BinaryOperator>(I)) {
684 if (BO->getOpcode() == Instruction::Add ||
686 CI = dyn_cast<ConstantInt>(BO->getOperand(1));
687 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(I)) {
688 Value *LastOperand =
GEP->getOperand(
GEP->getNumOperands()-1);
694 BaseUsers.push_back(II);
697 DEBUG(
dbgs() <<
"LRR: Aborting due to non-instruction: " << *I <<
"\n");
703 if (Roots.find(V) != Roots.end())
709 DEBUG(
dbgs() <<
"LRR: Aborting due to negative value: " << V <<
"\n");
713 Roots[V] = cast<Instruction>(
I);
722 if (BaseUsers.size()) {
723 if (Roots.find(0) != Roots.end()) {
724 DEBUG(
dbgs() <<
"LRR: Multiple roots found for base - aborting!\n");
731 unsigned NumBaseUses = BaseUsers.size();
732 if (NumBaseUses == 0)
733 NumBaseUses = Roots.begin()->second->getNumUses();
736 for (
auto &KV : Roots) {
739 if (KV.second->getNumUses() != NumBaseUses) {
740 DEBUG(
dbgs() <<
"LRR: Aborting - Root and Base #users not the same: "
741 <<
"#Base=" << NumBaseUses <<
", #Root=" <<
742 KV.second->getNumUses() <<
"\n");
750 bool LoopReroll::DAGRootTracker::
751 findRootsRecursive(
Instruction *I, SmallInstructionSet SubsumedInsts) {
757 if ((I->
getOpcode() == Instruction::Mul ||
760 findRootsBase(I, SubsumedInsts))
763 SubsumedInsts.insert(I);
765 for (User *V : I->
users()) {
767 if (std::find(LoopIncs.begin(), LoopIncs.end(),
I) != LoopIncs.end())
771 !findRootsRecursive(I, SubsumedInsts))
777 bool LoopReroll::DAGRootTracker::
778 findRootsBase(
Instruction *IVU, SmallInstructionSet SubsumedInsts) {
782 if (IVU->
getOpcode() != Instruction::Mul &&
786 std::map<int64_t, Instruction*> V;
787 if (!collectPossibleRoots(IVU, V))
792 if (V.find(0) == V.end())
793 SubsumedInsts.insert(IVU);
797 DRS.BaseInst =
nullptr;
801 DRS.BaseInst = KV.second;
802 DRS.SubsumedInsts = SubsumedInsts;
803 }
else if (DRS.Roots.empty()) {
804 DRS.Roots.push_back(KV.second);
805 }
else if (V.find(KV.first - 1) != V.end()) {
806 DRS.Roots.push_back(KV.second);
809 RootSets.push_back(DRS);
810 DRS.BaseInst = KV.second;
811 DRS.SubsumedInsts = SubsumedInsts;
815 RootSets.push_back(DRS);
820 bool LoopReroll::DAGRootTracker::findRoots() {
822 const SCEVAddRecExpr *RealIVSCEV = cast<SCEVAddRecExpr>(SE->getSCEV(IV));
823 Inc = cast<SCEVConstant>(RealIVSCEV->
getOperand(1))->
824 getValue()->getZExtValue();
826 assert(RootSets.empty() &&
"Unclean state!");
828 for (
auto *IVU : IV->
users()) {
830 LoopIncs.push_back(cast<Instruction>(IVU));
832 if (!findRootsRecursive(IV, SmallInstructionSet()))
834 LoopIncs.push_back(IV);
836 if (!findRootsBase(IV, SmallInstructionSet()))
841 if (RootSets.empty()) {
842 DEBUG(
dbgs() <<
"LRR: Aborting because no root sets found!\n");
845 for (
auto &V : RootSets) {
846 if (V.Roots.empty() || V.Roots.size() != RootSets[0].Roots.size()) {
848 <<
"LRR: Aborting because not all root sets have the same size\n");
855 for (
auto &V : RootSets) {
870 unsigned N = V.Roots.size() + 1;
871 const SCEV *StepSCEV = SE->getMinusSCEV(SE->getSCEV(V.Roots[0]), ADR);
872 const SCEV *ScaleSCEV = SE->getConstant(StepSCEV->
getType(),
N);
873 if (ADR->getStepRecurrence(*SE) != SE->getMulExpr(StepSCEV, ScaleSCEV)) {
874 DEBUG(
dbgs() <<
"LRR: Aborting because iterations are not consecutive\n");
878 Scale = RootSets[0].Roots.size() + 1;
880 if (Scale > IL_MaxRerollIterations) {
881 DEBUG(
dbgs() <<
"LRR: Aborting - too many iterations found. "
882 <<
"#Found=" << Scale <<
", #Max=" << IL_MaxRerollIterations
887 DEBUG(
dbgs() <<
"LRR: Successfully found roots: Scale=" << Scale <<
"\n");
892 bool LoopReroll::DAGRootTracker::collectUsedInstructions(SmallInstructionSet &PossibleRedSet) {
896 Uses[&
I].resize(IL_End);
899 SmallInstructionSet Exclude;
900 for (
auto &DRS : RootSets) {
901 Exclude.insert(DRS.Roots.begin(), DRS.Roots.end());
902 Exclude.insert(DRS.SubsumedInsts.begin(), DRS.SubsumedInsts.end());
903 Exclude.insert(DRS.BaseInst);
905 Exclude.insert(LoopIncs.begin(), LoopIncs.end());
907 for (
auto &DRS : RootSets) {
909 collectInLoopUserSet(DRS.BaseInst, Exclude, PossibleRedSet, VBase);
910 for (
auto *I : VBase) {
915 for (
auto *Root : DRS.Roots) {
917 collectInLoopUserSet(Root, Exclude, PossibleRedSet, V);
920 if (V.
size() != VBase.size()) {
921 DEBUG(
dbgs() <<
"LRR: Aborting - use sets are different sizes\n");
932 for (
auto *I : DRS.SubsumedInsts) {
940 for (
auto &DRS : RootSets) {
941 Exclude.insert(DRS.Roots.begin(), DRS.Roots.end());
942 Exclude.insert(DRS.SubsumedInsts.begin(), DRS.SubsumedInsts.end());
943 Exclude.insert(DRS.BaseInst);
947 collectInLoopUserSet(LoopIncs, Exclude, PossibleRedSet, V);
960 LoopReroll::DAGRootTracker::nextInstr(
int Val, UsesTy &
In,
961 const SmallInstructionSet &Exclude,
962 UsesTy::iterator *StartI) {
963 UsesTy::iterator I = StartI ? *StartI : In.begin();
964 while (I != In.end() && (I->second.test(Val) == 0 ||
965 Exclude.count(I->first) != 0))
970 bool LoopReroll::DAGRootTracker::isBaseInst(
Instruction *I) {
971 for (
auto &DRS : RootSets) {
972 if (DRS.BaseInst == I)
978 bool LoopReroll::DAGRootTracker::isRootInst(
Instruction *I) {
979 for (
auto &DRS : RootSets) {
980 if (std::find(DRS.Roots.begin(), DRS.Roots.end(),
I) != DRS.Roots.end())
988 bool LoopReroll::DAGRootTracker::instrDependsOn(
Instruction *I,
989 UsesTy::iterator Start,
990 UsesTy::iterator End) {
991 for (
auto *U : I->
users()) {
992 for (
auto It = Start; It != End; ++It)
999 bool LoopReroll::DAGRootTracker::validate(ReductionTracker &Reductions) {
1014 SmallInstructionSet PossibleRedSet;
1015 SmallInstructionSet PossibleRedLastSet;
1016 SmallInstructionSet PossibleRedPHISet;
1017 Reductions.restrictToScale(Scale, PossibleRedSet,
1018 PossibleRedPHISet, PossibleRedLastSet);
1021 if (!collectUsedInstructions(PossibleRedSet))
1025 for (
auto *I : PossibleRedPHISet) {
1026 Uses[
I].set(IL_All);
1031 for (
auto &KV : Uses) {
1032 if (KV.second.count() != 1) {
1033 DEBUG(
dbgs() <<
"LRR: Aborting - instruction is not used in 1 iteration: "
1034 << *KV.first <<
" (#uses=" << KV.second.count() <<
")\n");
1040 for (
auto &KV : Uses) {
1041 dbgs() <<
"LRR: " << KV.second.find_first() <<
"\t" << *KV.first <<
"\n";
1045 for (
unsigned Iter = 1; Iter < Scale; ++Iter) {
1050 bool FutureSideEffects =
false;
1056 SmallInstructionSet Visited;
1057 auto BaseIt = nextInstr(0, Uses, Visited);
1058 auto RootIt = nextInstr(Iter, Uses, Visited);
1059 auto LastRootIt = Uses.begin();
1061 while (BaseIt != Uses.end() && RootIt != Uses.end()) {
1066 bool Continue =
false;
1067 if (isBaseInst(BaseInst)) {
1068 Visited.insert(BaseInst);
1069 BaseIt = nextInstr(0, Uses, Visited);
1072 if (isRootInst(RootInst)) {
1073 LastRootIt = RootIt;
1074 Visited.insert(RootInst);
1075 RootIt = nextInstr(Iter, Uses, Visited);
1078 if (Continue)
continue;
1091 auto TryIt = RootIt;
1093 while (TryIt != Uses.end() &&
1097 TryIt = nextInstr(Iter, Uses, Visited, &TryIt);
1100 if (TryIt == Uses.end() || TryIt == RootIt ||
1101 instrDependsOn(TryIt->first, RootIt, TryIt)) {
1102 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1103 " vs. " << *RootInst <<
"\n");
1108 RootInst = TryIt->first;
1119 for (; LastRootIt < RootIt; ++LastRootIt) {
1121 if (LastRootIt->second.find_first() < (
int)Iter)
1132 FutureSideEffects =
true;
1138 if (RootIt->second.count() > 1) {
1139 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1140 " vs. " << *RootInst <<
" (prev. case overlap)\n");
1148 for (
auto &K : AST) {
1149 if (K.aliasesUnknownInst(RootInst, *AA)) {
1150 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1151 " vs. " << *RootInst <<
" (depends on future store)\n");
1164 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1165 " vs. " << *RootInst <<
1166 " (side effects prevent reordering)\n");
1180 bool InReduction = Reductions.isPairInSame(BaseInst, RootInst);
1183 bool Swapped =
false, SomeOpMatched =
false;
1191 if (
Instruction *Op2I = dyn_cast<Instruction>(Op2))
1192 if (Reductions.isPairInSame(RootInst, Op2I))
1196 if (BMI != BaseMap.
end()) {
1199 for (
auto &DRS : RootSets) {
1207 if (BaseInst->
getOperand(Swapped ?
unsigned(!j) : j) != Op2) {
1213 if (!Swapped && BaseInst->
isCommutative() && !SomeOpMatched &&
1217 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst
1218 <<
" vs. " << *RootInst <<
" (operand " << j <<
")\n");
1223 SomeOpMatched =
true;
1227 if ((!PossibleRedLastSet.count(BaseInst) &&
1229 (!PossibleRedLastSet.count(RootInst) &&
1231 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1232 " vs. " << *RootInst <<
" (uses outside loop)\n");
1236 Reductions.recordPair(BaseInst, RootInst, Iter);
1237 BaseMap.
insert(std::make_pair(RootInst, BaseInst));
1239 LastRootIt = RootIt;
1240 Visited.insert(BaseInst);
1241 Visited.insert(RootInst);
1242 BaseIt = nextInstr(0, Uses, Visited);
1243 RootIt = nextInstr(Iter, Uses, Visited);
1245 assert (BaseIt == Uses.end() && RootIt == Uses.end() &&
1246 "Mismatched set sizes!");
1249 DEBUG(
dbgs() <<
"LRR: Matched all iteration increments for " <<
1255 void LoopReroll::DAGRootTracker::replace(
const SCEV *IterCount) {
1259 J != Header->
rend();) {
1260 unsigned I = Uses[&*J].find_first();
1261 if (I > 0 && I < IL_All) {
1263 DEBUG(
dbgs() <<
"LRR: removing: " << *D <<
"\n");
1273 for (
auto &DRS : RootSets) {
1276 cast<SCEVAddRecExpr>(SE->getSCEV(DRS.BaseInst));
1279 (SE->getAddRecExpr(Start,
1280 SE->getConstant(RealIVSCEV->
getType(), 1),
1286 for (
auto &KV : Uses) {
1287 if (KV.second.find_first() == 0)
1288 KV.first->replaceUsesOfWith(DRS.BaseInst, NewIV);
1293 if (Uses[BI].find_first() == IL_All) {
1297 const SCEV *ICMinus1SCEV =
1298 SE->getMinusSCEV(ICSCEV, SE->getConstant(ICSCEV->
getType(), 1));
1301 if (isa<SCEVConstant>(ICMinus1SCEV)) {
1302 ICMinus1 = Expander.expandCodeFor(ICMinus1SCEV, NewIV->
getType(), BI);
1308 ICMinus1 = Expander.expandCodeFor(ICMinus1SCEV, NewIV->
getType(),
1314 BI->setCondition(Cond);
1316 if (BI->getSuccessor(1) != Header)
1317 BI->swapSuccessors();
1330 bool LoopReroll::ReductionTracker::validateSelected() {
1335 int PrevIter = 0, BaseCount = 0, Count = 0;
1340 int Iter = PossibleRedIter[J];
1341 if (Iter != PrevIter && Iter != PrevIter + 1 &&
1342 !PossibleReds[i].getReducedValue()->isAssociative()) {
1343 DEBUG(
dbgs() <<
"LRR: Out-of-order non-associative reduction: " <<
1348 if (Iter != PrevIter) {
1349 if (Count != BaseCount) {
1350 DEBUG(
dbgs() <<
"LRR: Iteration " << PrevIter <<
1351 " reduction use count " << Count <<
1352 " is not equal to the base use count " <<
1375 void LoopReroll::ReductionTracker::replaceSelected() {
1382 for (
int e = PossibleReds[i].
size(); j != e; ++j)
1383 if (PossibleRedIter[PossibleReds[i][j]] != 0) {
1389 SmallInstructionVector
Users;
1390 for (User *U : PossibleReds[i].getReducedValue()->
users()) {
1391 Users.push_back(cast<Instruction>(U));
1394 for (SmallInstructionVector::iterator J = Users.begin(),
1395 JE = Users.end(); J != JE; ++J)
1396 (*J)->replaceUsesOfWith(PossibleReds[i].getReducedValue(),
1397 PossibleReds[i][j]);
1445 const SCEV *IterCount,
1446 ReductionTracker &Reductions) {
1447 DAGRootTracker DAGRoots(
this, L, IV, SE, AA, TLI);
1449 if (!DAGRoots.findRoots())
1451 DEBUG(
dbgs() <<
"LRR: Found all root induction increments for: " <<
1454 if (!DAGRoots.validate(Reductions))
1456 if (!Reductions.validateSelected())
1461 Reductions.replaceSelected();
1462 DAGRoots.replace(IterCount);
1469 if (skipOptnoneFunction(L))
1472 AA = &getAnalysis<AliasAnalysis>();
1473 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1474 SE = &getAnalysis<ScalarEvolution>();
1475 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
1476 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1480 "] Loop %" << Header->
getName() <<
" (" <<
1483 bool Changed =
false;
1489 if (!SE->hasLoopInvariantBackedgeTakenCount(L))
1492 const SCEV *LIBETC = SE->getBackedgeTakenCount(L);
1493 const SCEV *IterCount =
1494 SE->getAddExpr(LIBETC, SE->getConstant(LIBETC->
getType(), 1));
1495 DEBUG(
dbgs() <<
"LRR: iteration count = " << *IterCount <<
"\n");
1499 SmallInstructionVector PossibleIVs;
1500 collectPossibleIVs(L, PossibleIVs);
1502 if (PossibleIVs.empty()) {
1503 DEBUG(
dbgs() <<
"LRR: No possible IVs found\n");
1507 ReductionTracker Reductions;
1508 collectPossibleReductions(L, Reductions);
1512 for (SmallInstructionVector::iterator I = PossibleIVs.begin(),
1513 IE = PossibleIVs.end(); I !=
IE; ++
I)
1514 if (reroll(*I, L, Header, IterCount, Reductions)) {
Pass interface - Implemented by all 'passes'.
VectorType::iterator iterator
const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
evaluateAtIteration - Return the value of this chain of recurrences at the specified iteration number...
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
const_iterator end(StringRef path)
Get end iterator over path.
Pass * createLoopRerollPass()
static bool isSimpleLoadStore(Instruction *I)
iterator_range< use_iterator > uses()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
DenseSet - This implements a dense probed hash-table based set.
unsigned getNumOperands() const
ScalarEvolution - This class is the main scalar evolution driver.
const_iterator begin(StringRef path)
Get begin iterator over path.
This class implements a map that also provides access to all stored values in a deterministic order...
const Function * getParent() const
Return the enclosing method, or null if none.
bool SimplifyInstructionsInBlock(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
SimplifyInstructionsInBlock - Scan the specified basic block and try to simplify any instructions in ...
void initializeLoopRerollPass(PassRegistry &)
LoadInst - an instruction for reading from memory.
reverse_iterator rbegin()
iv Induction Variable Users
BlockT * getHeader() const
const SCEV * getStart() const
StringRef getName() const
Return a constant reference to the value's name.
iterator begin()
Instruction iterator methods.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
static cl::opt< unsigned > MaxInc("max-reroll-increment", cl::init(2048), cl::Hidden, cl::desc("The maximum increment for loop rerolling"))
BasicBlock * InsertPreheaderForLoop(Loop *L, Pass *P)
InsertPreheaderForLoop - Once we discover that a loop doesn't have a preheader, this method is called...
const APInt & getValue() const
Return the constant as an APInt value reference.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
A Use represents the edge between a Value definition and its users.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
static bool hasUsesOutsideLoop(Instruction *I, Loop *L)
bool mayReadFromMemory() const
mayReadFromMemory - Return true if this instruction may read memory.
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
bool isAssociative() const
isAssociative - Return true if the instruction is associative:
static bool add(uint64_t *dest, const uint64_t *x, const uint64_t *y, unsigned len)
This function adds the integer array x to the integer array Y and places the result in dest...
bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
DeleteDeadPHIs - Examine each PHI in the given block and delete it if it is dead. ...
StoreInst - an instruction for storing to memory.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
InstListType::reverse_iterator reverse_iterator
initializer< Ty > init(const Ty &Val)
friend const_iterator end(StringRef path)
Get end iterator over path.
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
Type * getType() const
getType - Return the LLVM type of this SCEV expression.
BranchInst - Conditional or Unconditional Branch instruction.
const SCEV * getOperand(unsigned i) const
int64_t getSExtValue() const
Get sign extended value.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
contains - Return true if the specified loop is contained within in this loop.
This instruction compares its operands according to the predicate given to the constructor.
Value * getOperand(unsigned i) const
static bool isLoopIncrement(User *U, Instruction *IV)
bool isCommutative() const
isCommutative - Return true if the instruction is commutative:
#define INITIALIZE_AG_DEPENDENCY(depName)
bool mayWriteToMemory() const
mayWriteToMemory - Return true if this instruction may modify memory.
BinaryOps getOpcode() const
std::pair< iterator, bool > insert(const ValueT &V)
MemIntrinsic - This is the common base class for memset/memcpy/memmove.
This is the shared class of boolean and integer constants.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static cl::opt< unsigned > NumToleratedFailedMatches("reroll-num-tolerated-failed-matches", cl::init(400), cl::Hidden, cl::desc("The maximum number of failures to tolerate"" during fuzzy matching. (default: 400)"))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
iterator_range< user_iterator > users()
This class uses information about analyze scalars to rewrite expressions in canonical form...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static bool isSimpleArithmeticOp(User *IVU)
Return true if IVU is a "simple" arithmetic operation.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
SCEV - This class represents an analyzed expression in the program.
unsigned getNumBlocks() const
getNumBlocks - Get the number of blocks in this loop in constant time.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool hasOneUse() const
Return true if there is exactly one user of this value.
iterator find(const KeyT &Val)
user_iterator user_begin()
LLVM Value Representation.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
The legacy pass manager's analysis pass to compute loop information.
C - The default llvm calling convention, compatible with C.
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
isSafeToSpeculativelyExecute - Return true if the instruction does not have any effects besides calcu...
Legacy analysis pass which computes a DominatorTree.
bool isSameOperationAs(const Instruction *I, unsigned flags=0) const
This function determines if the specified instruction executes the same operation as the current one...
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
unsigned getNumUses() const
This method computes the number of uses of this Value.
SCEVConstant - This class represents a constant integer value.