40 #define DEBUG_TYPE "loop-reroll"
42 STATISTIC(NumRerolledLoops,
"Number of rerolled loops");
46 cl::desc(
"The maximum increment for loop rerolling"));
51 cl::desc(
"The maximum number of failures to tolerate"
52 " during fuzzy matching. (default: 400)"));
132 IL_MaxRerollIterations = 32,
139 class LoopReroll :
public LoopPass {
173 struct SimpleLoopReduction {
175 : Valid(
false), Instructions(1, P) {
176 assert(isa<PHINode>(P) &&
"First reduction instruction must be a PHI");
185 assert(Valid &&
"Using invalid reduction");
186 return Instructions.front();
190 assert(Valid &&
"Using invalid reduction");
191 return Instructions.back();
195 assert(Valid &&
"Using invalid reduction");
196 return Instructions[
i+1];
202 size_t size()
const {
203 assert(Valid &&
"Using invalid reduction");
204 return Instructions.size()-1;
207 typedef SmallInstructionVector::iterator iterator;
208 typedef SmallInstructionVector::const_iterator const_iterator;
211 assert(Valid &&
"Using invalid reduction");
212 return std::next(Instructions.begin());
215 const_iterator
begin()
const {
216 assert(Valid &&
"Using invalid reduction");
217 return std::next(Instructions.begin());
220 iterator
end() {
return Instructions.
end(); }
221 const_iterator
end()
const {
return Instructions.
end(); }
225 SmallInstructionVector Instructions;
232 struct ReductionTracker {
236 void addSLR(SimpleLoopReduction &SLR) { PossibleReds.
push_back(SLR); }
246 void restrictToScale(uint64_t Scale,
247 SmallInstructionSet &PossibleRedSet,
248 SmallInstructionSet &PossibleRedPHISet,
249 SmallInstructionSet &PossibleRedLastSet) {
250 PossibleRedIdx.clear();
251 PossibleRedIter.clear();
254 for (
unsigned i = 0, e = PossibleReds.size();
i != e; ++
i)
255 if (PossibleReds[
i].size() % Scale == 0) {
256 PossibleRedLastSet.insert(PossibleReds[
i].getReducedValue());
257 PossibleRedPHISet.insert(PossibleReds[
i].getPHI());
259 PossibleRedSet.insert(PossibleReds[
i].getPHI());
260 PossibleRedIdx[PossibleReds[
i].getPHI()] =
i;
262 PossibleRedSet.insert(J);
263 PossibleRedIdx[J] =
i;
274 if (J1I != PossibleRedIdx.
end()) {
276 if (J2I != PossibleRedIdx.
end() && J1I->second == J2I->second)
287 if (PossibleRedIdx.count(J1)) {
288 assert(PossibleRedIdx.count(J2) &&
289 "Recording reduction vs. non-reduction instruction?");
291 PossibleRedIter[J1] = 0;
292 PossibleRedIter[J2] =
i;
294 int Idx = PossibleRedIdx[J1];
295 assert(Idx == PossibleRedIdx[J2] &&
296 "Recording pair from different reductions?");
304 bool validateSelected();
305 void replaceSelected();
309 SmallReductionVector PossibleReds;
344 SmallInstructionVector Roots;
346 SmallInstructionSet SubsumedInsts;
351 struct DAGRootTracker {
358 : Parent(Parent), L(L), SE(SE), AA(AA), TLI(TLI), DT(DT), LI(LI),
359 PreserveLCSSA(PreserveLCSSA), IV(IV), IVToIncMap(IncrMap),
360 LoopControlIV(LoopCtrlIV) {}
365 bool validate(ReductionTracker &Reductions);
369 void replace(
const SCEV *IterCount);
375 SmallInstructionSet SubsumedInsts);
376 bool findRootsBase(
Instruction *IVU, SmallInstructionSet SubsumedInsts);
378 std::map<int64_t,Instruction*> &Roots);
379 bool validateRootSet(DAGRootSet &DRS);
381 bool collectUsedInstructions(SmallInstructionSet &PossibleRedSet);
382 void collectInLoopUserSet(
const SmallInstructionVector &Roots,
383 const SmallInstructionSet &Exclude,
384 const SmallInstructionSet &Final,
387 const SmallInstructionSet &Exclude,
388 const SmallInstructionSet &Final,
391 UsesTy::iterator nextInstr(
int Val, UsesTy &
In,
392 const SmallInstructionSet &Exclude,
393 UsesTy::iterator *StartI=
nullptr);
397 UsesTy::iterator Start,
398 UsesTy::iterator
End);
400 void updateNonLoopCtrlIncr();
424 SmallInstructionVector LoopIncs;
437 if (!isa<BranchInst>(TI) || !isa<CmpInst>(I))
439 return I->
hasOneUse() && TI->getOperand(0) ==
I;
443 void collectPossibleIVs(
Loop *L, SmallInstructionVector &PossibleIVs);
444 void collectPossibleReductions(
Loop *L,
445 ReductionTracker &Reductions);
447 ReductionTracker &Reductions);
458 return new LoopReroll;
466 if (!L->
contains(cast<Instruction>(U)))
473 const SCEV *SCEVExpr,
480 if (
const SCEVConstant *IncSCEV = dyn_cast<SCEVConstant>(SCEVExpr)) {
483 const SCEV *SizeOfExpr =
485 if (IncSCEV->getValue()->getValue().isNegative()) {
486 const SCEV *NewSCEV =
505 if (!
Unknown->isSizeOf(AllocTy))
522 if (IVUses != 2 && IVUses != 1)
527 bool IsCompInst = isCompareUsedByBranch(cast<Instruction>(
User));
530 if (IncOrCmpUses != 2 && IncOrCmpUses != 1)
537 if (IsCompInst || IncOrCmpUses != 2)
542 if (IVUses == 2 && IncOrCmpUses != 1)
546 if (
auto *BO = dyn_cast<BinaryOperator>(
User)) {
551 if (
PHINode *PN = dyn_cast<PHINode>(UU)) {
560 if (BO->hasNoSignedWrap() && UUser && UUser->
getNumUses() == 1 &&
561 isa<SExtInst>(UUser))
563 if (!isCompareUsedByBranch(UUser))
570 }
else if (!IsCompInst)
578 void LoopReroll::collectPossibleIVs(
Loop *L,
579 SmallInstructionVector &PossibleIVs) {
583 if (!isa<PHINode>(I))
589 dyn_cast<SCEVAddRecExpr>(SE->getSCEV(&*I))) {
590 if (PHISCEV->getLoop() !=
L)
592 if (!PHISCEV->isAffine())
605 DEBUG(
dbgs() <<
"LRR: Possible IV: " << *I <<
" = " << *PHISCEV
608 if (isLoopControlIV(L, &*I)) {
609 assert(!LoopControlIV &&
"Found two loop control only IV");
610 LoopControlIV = &(*I);
611 DEBUG(
dbgs() <<
"LRR: Possible loop control only IV: " << *I <<
" = "
612 << *PHISCEV <<
"\n");
614 PossibleIVs.push_back(&*I);
624 assert(!Valid &&
"Cannot add to an already-valid chain");
639 if (!(isa<PHINode>(Instructions.back()) ||
643 Instructions.push_back(C);
647 if (Instructions.size() < 2 ||
655 if (L->
contains(cast<Instruction>(U)))
656 if (cast<Instruction>(U) != Instructions.front())
660 Instructions.push_back(C);
665 void LoopReroll::collectPossibleReductions(
Loop *L,
666 ReductionTracker &Reductions) {
670 if (!isa<PHINode>(I))
675 SimpleLoopReduction SLR(&*I, L);
679 DEBUG(
dbgs() <<
"LRR: Possible reduction: " << *I <<
" (with " <<
680 SLR.size() <<
" chained instructions)\n");
681 Reductions.addSLR(SLR);
697 void LoopReroll::DAGRootTracker::collectInLoopUserSet(
698 Instruction *Root,
const SmallInstructionSet &Exclude,
699 const SmallInstructionSet &Final,
701 SmallInstructionVector
Queue(1, Root);
702 while (!
Queue.empty()) {
704 if (!Users.
insert(I).second)
710 if (
PHINode *PN = dyn_cast<PHINode>(User)) {
712 if (PN->getIncomingBlock(U) == L->
getHeader())
716 if (L->
contains(User) && !Exclude.count(User)) {
717 Queue.push_back(User);
723 OIE = I->
op_end(); OI != OIE; ++OI) {
734 void LoopReroll::DAGRootTracker::collectInLoopUserSet(
735 const SmallInstructionVector &Roots,
736 const SmallInstructionSet &Exclude,
737 const SmallInstructionSet &Final,
740 collectInLoopUserSet(Root, Exclude, Final, Users);
744 if (
LoadInst *LI = dyn_cast<LoadInst>(I))
745 return LI->isUnordered();
747 return SI->isUnordered();
749 return !
MI->isVolatile();
759 default:
return false;
761 case Instruction::Sub:
762 case Instruction::Mul:
763 case Instruction::Shl:
764 case Instruction::AShr:
765 case Instruction::LShr:
766 case Instruction::GetElementPtr:
767 case Instruction::Trunc:
768 case Instruction::ZExt:
769 case Instruction::SExt:
780 (!BO && !isa<GetElementPtrInst>(U)))
783 for (
auto *UU : U->
users()) {
791 bool LoopReroll::DAGRootTracker::
792 collectPossibleRoots(
Instruction *Base, std::map<int64_t,Instruction*> &Roots) {
793 SmallInstructionVector BaseUsers;
795 for (
auto *I : Base->
users()) {
799 LoopIncs.push_back(cast<Instruction>(I));
804 if (
auto *BO = dyn_cast<BinaryOperator>(I)) {
807 CI = dyn_cast<ConstantInt>(BO->getOperand(1));
808 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(I)) {
809 Value *LastOperand =
GEP->getOperand(
GEP->getNumOperands()-1);
815 BaseUsers.push_back(II);
818 DEBUG(
dbgs() <<
"LRR: Aborting due to non-instruction: " << *I <<
"\n");
824 if (Roots.find(V) != Roots.end())
828 Roots[V] = cast<Instruction>(
I);
832 if (Roots.empty() || (Roots.size() == 1 && BaseUsers.empty()))
838 if (BaseUsers.size()) {
839 if (Roots.find(0) != Roots.end()) {
840 DEBUG(
dbgs() <<
"LRR: Multiple roots found for base - aborting!\n");
847 unsigned NumBaseUses = BaseUsers.size();
848 if (NumBaseUses == 0)
849 NumBaseUses = Roots.begin()->second->getNumUses();
852 for (
auto &KV : Roots) {
855 if (KV.second->getNumUses() != NumBaseUses) {
856 DEBUG(
dbgs() <<
"LRR: Aborting - Root and Base #users not the same: "
857 <<
"#Base=" << NumBaseUses <<
", #Root=" <<
858 KV.second->getNumUses() <<
"\n");
866 void LoopReroll::DAGRootTracker::
867 findRootsRecursive(
Instruction *I, SmallInstructionSet SubsumedInsts) {
873 if (I != IV && findRootsBase(I, SubsumedInsts))
876 SubsumedInsts.insert(I);
878 for (User *V : I->
users()) {
887 findRootsRecursive(I, SubsumedInsts);
891 bool LoopReroll::DAGRootTracker::validateRootSet(DAGRootSet &DRS) {
892 if (DRS.Roots.empty())
907 unsigned N = DRS.Roots.size() + 1;
908 const SCEV *StepSCEV = SE->getMinusSCEV(SE->getSCEV(DRS.Roots[0]), ADR);
909 const SCEV *ScaleSCEV = SE->getConstant(StepSCEV->
getType(),
N);
910 if (ADR->getStepRecurrence(*SE) != SE->getMulExpr(StepSCEV, ScaleSCEV))
916 bool LoopReroll::DAGRootTracker::
917 findRootsBase(
Instruction *IVU, SmallInstructionSet SubsumedInsts) {
920 if (!IVU_ADR || IVU_ADR->getLoop() !=
L)
923 std::map<int64_t, Instruction*> V;
924 if (!collectPossibleRoots(IVU, V))
929 if (V.find(0) == V.end())
930 SubsumedInsts.insert(IVU);
934 DRS.BaseInst =
nullptr;
940 DRS.BaseInst = KV.second;
941 DRS.SubsumedInsts = SubsumedInsts;
942 }
else if (DRS.Roots.empty()) {
943 DRS.Roots.push_back(KV.second);
944 }
else if (V.find(KV.first - 1) != V.end()) {
945 DRS.Roots.push_back(KV.second);
948 if (!validateRootSet(DRS))
953 DRS.BaseInst = KV.second;
958 if (!validateRootSet(DRS))
963 RootSets.append(PotentialRootSets.
begin(), PotentialRootSets.
end());
968 bool LoopReroll::DAGRootTracker::findRoots() {
969 Inc = IVToIncMap[IV];
971 assert(RootSets.empty() &&
"Unclean state!");
973 for (
auto *IVU : IV->
users()) {
975 LoopIncs.push_back(cast<Instruction>(IVU));
977 findRootsRecursive(IV, SmallInstructionSet());
978 LoopIncs.push_back(IV);
980 if (!findRootsBase(IV, SmallInstructionSet()))
985 if (RootSets.empty()) {
986 DEBUG(
dbgs() <<
"LRR: Aborting because no root sets found!\n");
989 for (
auto &V : RootSets) {
990 if (V.Roots.empty() || V.Roots.size() != RootSets[0].Roots.size()) {
992 <<
"LRR: Aborting because not all root sets have the same size\n");
997 Scale = RootSets[0].Roots.size() + 1;
999 if (Scale > IL_MaxRerollIterations) {
1000 DEBUG(
dbgs() <<
"LRR: Aborting - too many iterations found. "
1001 <<
"#Found=" << Scale <<
", #Max=" << IL_MaxRerollIterations
1006 DEBUG(
dbgs() <<
"LRR: Successfully found roots: Scale=" << Scale <<
"\n");
1011 bool LoopReroll::DAGRootTracker::collectUsedInstructions(SmallInstructionSet &PossibleRedSet) {
1015 Uses[&
I].resize(IL_End);
1018 SmallInstructionSet Exclude;
1019 for (
auto &DRS : RootSets) {
1020 Exclude.insert(DRS.Roots.begin(), DRS.Roots.end());
1021 Exclude.insert(DRS.SubsumedInsts.begin(), DRS.SubsumedInsts.end());
1022 Exclude.insert(DRS.BaseInst);
1024 Exclude.insert(LoopIncs.begin(), LoopIncs.end());
1026 for (
auto &DRS : RootSets) {
1028 collectInLoopUserSet(DRS.BaseInst, Exclude, PossibleRedSet, VBase);
1029 for (
auto *I : VBase) {
1034 for (
auto *Root : DRS.Roots) {
1036 collectInLoopUserSet(Root, Exclude, PossibleRedSet, V);
1039 if (V.
size() != VBase.size()) {
1040 DEBUG(
dbgs() <<
"LRR: Aborting - use sets are different sizes\n");
1051 for (
auto *I : DRS.SubsumedInsts) {
1052 Uses[
I].set(IL_All);
1059 for (
auto &DRS : RootSets) {
1060 Exclude.insert(DRS.Roots.begin(), DRS.Roots.end());
1061 Exclude.insert(DRS.SubsumedInsts.begin(), DRS.SubsumedInsts.end());
1062 Exclude.insert(DRS.BaseInst);
1066 collectInLoopUserSet(LoopIncs, Exclude, PossibleRedSet, V);
1068 Uses[
I].set(IL_All);
1079 LoopReroll::DAGRootTracker::nextInstr(
int Val, UsesTy &
In,
1080 const SmallInstructionSet &Exclude,
1081 UsesTy::iterator *StartI) {
1082 UsesTy::iterator I = StartI ? *StartI : In.begin();
1083 while (I != In.end() && (I->second.test(Val) == 0 ||
1084 Exclude.count(I->first) != 0))
1089 bool LoopReroll::DAGRootTracker::isBaseInst(
Instruction *I) {
1090 for (
auto &DRS : RootSets) {
1091 if (DRS.BaseInst == I)
1097 bool LoopReroll::DAGRootTracker::isRootInst(
Instruction *I) {
1098 for (
auto &DRS : RootSets) {
1107 bool LoopReroll::DAGRootTracker::instrDependsOn(
Instruction *I,
1108 UsesTy::iterator Start,
1109 UsesTy::iterator
End) {
1110 for (
auto *U : I->
users()) {
1111 for (
auto It = Start; It !=
End; ++It)
1119 if (isa<DbgInfoIntrinsic>(I))
1127 case llvm::Intrinsic::annotation:
1128 case Intrinsic::ptr_annotation:
1129 case Intrinsic::var_annotation:
1137 bool LoopReroll::DAGRootTracker::validate(ReductionTracker &Reductions) {
1152 SmallInstructionSet PossibleRedSet;
1153 SmallInstructionSet PossibleRedLastSet;
1154 SmallInstructionSet PossibleRedPHISet;
1155 Reductions.restrictToScale(Scale, PossibleRedSet,
1156 PossibleRedPHISet, PossibleRedLastSet);
1159 if (!collectUsedInstructions(PossibleRedSet))
1163 for (
auto *I : PossibleRedPHISet) {
1164 Uses[
I].set(IL_All);
1170 if (LoopControlIV && LoopControlIV != IV) {
1171 for (
auto *U : LoopControlIV->users()) {
1174 Uses[IVUser].set(IL_All);
1175 for (
auto *UU : IVUser->
users()) {
1178 Uses[UUser].set(IL_All);
1180 if (isa<SExtInst>(UUser)) {
1182 Uses[UUser].set(IL_All);
1185 if (UU->hasOneUse()) {
1188 Uses[BI].set(IL_All);
1196 for (
auto &KV : Uses) {
1198 DEBUG(
dbgs() <<
"LRR: Aborting - instruction is not used in 1 iteration: "
1199 << *KV.first <<
" (#uses=" << KV.second.count() <<
")\n");
1205 for (
auto &KV : Uses) {
1206 dbgs() <<
"LRR: " << KV.second.find_first() <<
"\t" << *KV.first <<
"\n";
1210 for (
unsigned Iter = 1; Iter < Scale; ++Iter) {
1215 bool FutureSideEffects =
false;
1221 SmallInstructionSet Visited;
1222 auto BaseIt = nextInstr(0, Uses, Visited);
1223 auto RootIt = nextInstr(Iter, Uses, Visited);
1224 auto LastRootIt = Uses.begin();
1226 while (BaseIt != Uses.end() && RootIt != Uses.end()) {
1231 bool Continue =
false;
1232 if (isBaseInst(BaseInst)) {
1233 Visited.insert(BaseInst);
1234 BaseIt = nextInstr(0, Uses, Visited);
1237 if (isRootInst(RootInst)) {
1238 LastRootIt = RootIt;
1239 Visited.insert(RootInst);
1240 RootIt = nextInstr(Iter, Uses, Visited);
1243 if (Continue)
continue;
1256 auto TryIt = RootIt;
1258 while (TryIt != Uses.end() &&
1262 TryIt = nextInstr(Iter, Uses, Visited, &TryIt);
1265 if (TryIt == Uses.end() || TryIt == RootIt ||
1266 instrDependsOn(TryIt->first, RootIt, TryIt)) {
1267 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1268 " vs. " << *RootInst <<
"\n");
1273 RootInst = TryIt->first;
1284 for (; LastRootIt < RootIt; ++LastRootIt) {
1286 if (LastRootIt->second.find_first() < (int)Iter)
1297 FutureSideEffects =
true;
1303 if (RootIt->second.count() > 1) {
1304 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1305 " vs. " << *RootInst <<
" (prev. case overlap)\n");
1313 for (
auto &K : AST) {
1314 if (K.aliasesUnknownInst(RootInst, *AA)) {
1315 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1316 " vs. " << *RootInst <<
" (depends on future store)\n");
1329 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1330 " vs. " << *RootInst <<
1331 " (side effects prevent reordering)\n");
1345 bool InReduction = Reductions.isPairInSame(BaseInst, RootInst);
1348 bool Swapped =
false, SomeOpMatched =
false;
1356 if (
Instruction *Op2I = dyn_cast<Instruction>(Op2))
1357 if (Reductions.isPairInSame(RootInst, Op2I))
1361 if (BMI != BaseMap.
end()) {
1364 for (
auto &DRS : RootSets) {
1372 if (BaseInst->
getOperand(Swapped ?
unsigned(!j) : j) != Op2) {
1378 if (!Swapped && BaseInst->
isCommutative() && !SomeOpMatched &&
1382 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst
1383 <<
" vs. " << *RootInst <<
" (operand " << j <<
")\n");
1388 SomeOpMatched =
true;
1392 if ((!PossibleRedLastSet.count(BaseInst) &&
1394 (!PossibleRedLastSet.count(RootInst) &&
1396 DEBUG(
dbgs() <<
"LRR: iteration root match failed at " << *BaseInst <<
1397 " vs. " << *RootInst <<
" (uses outside loop)\n");
1401 Reductions.recordPair(BaseInst, RootInst, Iter);
1402 BaseMap.
insert(std::make_pair(RootInst, BaseInst));
1404 LastRootIt = RootIt;
1405 Visited.insert(BaseInst);
1406 Visited.insert(RootInst);
1407 BaseIt = nextInstr(0, Uses, Visited);
1408 RootIt = nextInstr(Iter, Uses, Visited);
1410 assert (BaseIt == Uses.end() && RootIt == Uses.end() &&
1411 "Mismatched set sizes!");
1414 DEBUG(
dbgs() <<
"LRR: Matched all iteration increments for " <<
1420 void LoopReroll::DAGRootTracker::replace(
const SCEV *IterCount) {
1425 unsigned I = Uses[&*J].find_first();
1426 if (I > 0 && I < IL_All) {
1427 DEBUG(
dbgs() <<
"LRR: removing: " << *J <<
"\n");
1428 J++->eraseFromParent();
1435 bool HasTwoIVs = LoopControlIV && LoopControlIV != IV;
1438 updateNonLoopCtrlIncr();
1439 replaceIV(LoopControlIV, LoopControlIV, IterCount);
1442 for (
auto &DRS : RootSets)
1444 replaceIV(DRS.BaseInst, IV, IterCount);
1452 void LoopReroll::DAGRootTracker::updateNonLoopCtrlIncr() {
1453 const SCEV *NewInc =
nullptr;
1454 for (
auto *LoopInc : LoopIncs) {
1465 assert(COp &&
"Didn't find constant operand of LoopInc!\n");
1467 const APInt &AInt = COp->getValue()->getValue();
1468 const SCEV *ScaleSCEV = SE->getConstant(COp->getType(), Scale);
1470 NewInc = SE->getNegativeSCEV(COp);
1471 NewInc = SE->getUDivExpr(NewInc, ScaleSCEV);
1472 NewInc = SE->getNegativeSCEV(NewInc);
1474 NewInc = SE->getUDivExpr(COp, ScaleSCEV);
1476 LoopInc->setOperand(1, dyn_cast<SCEVConstant>(NewInc)->getValue());
1480 void LoopReroll::DAGRootTracker::replaceIV(
Instruction *Inst,
1482 const SCEV *IterCount) {
1484 int64_t Inc = IVToIncMap[InstIV];
1485 bool NeedNewIV = InstIV == LoopControlIV;
1486 bool Negative = !NeedNewIV && Inc < 0;
1488 const SCEVAddRecExpr *RealIVSCEV = cast<SCEVAddRecExpr>(SE->getSCEV(Inst));
1492 Start = SE->getConstant(Start->getType(), 0);
1494 const SCEV *SizeOfExpr =
nullptr;
1495 const SCEV *IncrExpr =
1496 SE->getConstant(RealIVSCEV->
getType(), Negative ? -1 : 1);
1497 if (
auto *PTy = dyn_cast<PointerType>(Inst->
getType())) {
1498 Type *ElTy = PTy->getElementType();
1500 SE->getSizeOfExpr(SE->getEffectiveSCEVType(Inst->
getType()), ElTy);
1501 IncrExpr = SE->getMulExpr(IncrExpr, SizeOfExpr);
1503 const SCEV *NewIVSCEV =
1509 Value *NewIV = Expander.expandCodeFor(NewIVSCEV, Inst->
getType(),
1512 for (
auto &KV : Uses)
1513 if (KV.second.find_first() == 0)
1514 KV.first->replaceUsesOfWith(Inst, NewIV);
1518 if (Uses[BI].find_first() == IL_All) {
1522 ICSCEV = SE->getMulExpr(IterCount,
1523 SE->getConstant(IterCount->
getType(), Scale));
1526 const SCEV *MinusPlus1SCEV =
1527 SE->getConstant(ICSCEV->
getType(), Negative ? -1 : 1);
1529 assert(SizeOfExpr &&
"SizeOfExpr is not initialized");
1530 MinusPlus1SCEV = SE->getMulExpr(MinusPlus1SCEV, SizeOfExpr);
1533 const SCEV *ICMinusPlus1SCEV = SE->getMinusSCEV(ICSCEV, MinusPlus1SCEV);
1536 if (isa<SCEVConstant>(ICMinusPlus1SCEV)) {
1545 if (!isa<PointerType>(NewIV->
getType()) && NeedNewIV &&
1546 (SE->getTypeSizeInBits(NewIV->
getType()) <
1547 SE->getTypeSizeInBits(ICMinusPlus1SCEV->
getType()))) {
1549 Builder.SetCurrentDebugLocation(BI->
getDebugLoc());
1550 NewIV = Builder.CreateSExt(NewIV, ICMinusPlus1SCEV->
getType());
1552 Value *ICMinusPlus1 = Expander.expandCodeFor(
1553 ICMinusPlus1SCEV, NewIV->
getType(), InsertPtr);
1557 BI->setCondition(Cond);
1559 if (BI->getSuccessor(1) != Header)
1560 BI->swapSuccessors();
1569 bool LoopReroll::ReductionTracker::validateSelected() {
1571 for (
int i : Reds) {
1572 int PrevIter = 0, BaseCount = 0, Count = 0;
1577 int Iter = PossibleRedIter[J];
1578 if (Iter != PrevIter && Iter != PrevIter + 1 &&
1579 !PossibleReds[i].getReducedValue()->isAssociative()) {
1580 DEBUG(
dbgs() <<
"LRR: Out-of-order non-associative reduction: " <<
1585 if (Iter != PrevIter) {
1586 if (Count != BaseCount) {
1587 DEBUG(
dbgs() <<
"LRR: Iteration " << PrevIter <<
1588 " reduction use count " << Count <<
1589 " is not equal to the base use count " <<
1612 void LoopReroll::ReductionTracker::replaceSelected() {
1615 for (
int i : Reds) {
1617 for (
int e = PossibleReds[i].size(); j != e; ++j)
1618 if (PossibleRedIter[PossibleReds[i][j]] != 0) {
1624 SmallInstructionVector
Users;
1625 for (User *U : PossibleReds[i].getReducedValue()->
users()) {
1626 Users.push_back(cast<Instruction>(U));
1631 PossibleReds[i][j]);
1679 const SCEV *IterCount,
1680 ReductionTracker &Reductions) {
1681 DAGRootTracker DAGRoots(
this, L, IV, SE, AA, TLI, DT, LI, PreserveLCSSA,
1682 IVToIncMap, LoopControlIV);
1684 if (!DAGRoots.findRoots())
1686 DEBUG(
dbgs() <<
"LRR: Found all root induction increments for: " <<
1689 if (!DAGRoots.validate(Reductions))
1691 if (!Reductions.validateSelected())
1696 Reductions.replaceSelected();
1697 DAGRoots.replace(IterCount);
1707 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1708 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1709 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1710 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
1711 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1712 PreserveLCSSA = mustPreserveAnalysisID(
LCSSAID);
1716 "] Loop %" << Header->
getName() <<
" (" <<
1723 if (!SE->hasLoopInvariantBackedgeTakenCount(L))
1726 const SCEV *LIBETC = SE->getBackedgeTakenCount(L);
1727 const SCEV *IterCount = SE->getAddExpr(LIBETC, SE->getOne(LIBETC->
getType()));
1729 DEBUG(
dbgs() <<
"LRR: iteration count = " << *IterCount <<
"\n");
1733 SmallInstructionVector PossibleIVs;
1735 LoopControlIV =
nullptr;
1736 collectPossibleIVs(L, PossibleIVs);
1738 if (PossibleIVs.empty()) {
1739 DEBUG(
dbgs() <<
"LRR: No possible IVs found\n");
1743 ReductionTracker Reductions;
1744 collectPossibleReductions(L, Reductions);
1745 bool Changed =
false;
1750 if (reroll(PossibleIV, L, Header, IterCount, Reductions)) {
Pass interface - Implemented by all 'passes'.
VectorType::iterator iterator
const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
Return the value of this chain of recurrences at the specified iteration number.
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
const_iterator end(StringRef path)
Get end iterator over path.
Pass * createLoopRerollPass()
iterator_range< use_iterator > uses()
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Implements a dense probed hash-table based set.
unsigned getNumOperands() const
The main scalar evolution driver.
BasicBlock * InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA)
InsertPreheaderForLoop - Once we discover that a loop doesn't have a preheader, this method is called...
const_iterator begin(StringRef path)
Get begin iterator over path.
This class implements a map that also provides access to all stored values in a deterministic order...
const Function * getParent() const
Return the enclosing method, or null if none.
bool SimplifyInstructionsInBlock(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
Scan the specified basic block and try to simplify any instructions in it and recursively delete dead...
void initializeLoopRerollPass(PassRegistry &)
An instruction for reading from memory.
reverse_iterator rbegin()
iv Induction Variable Users
Type * getElementType() const
BlockT * getHeader() const
const SCEV * getStart() const
StringRef getName() const
Return a constant reference to the value's name.
iterator begin()
Instruction iterator methods.
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
bool isNegative() const
Determine sign of this APInt.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Instruction * getFirstNonPHIOrDbg()
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
static bool isIgnorableInst(const Instruction *I)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
static cl::opt< unsigned > MaxInc("max-reroll-increment", cl::init(2048), cl::Hidden, cl::desc("The maximum increment for loop rerolling"))
const APInt & getValue() const
Return the constant as an APInt value reference.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
static bool isUnorderedLoadStore(Instruction *I)
A Use represents the edge between a Value definition and its users.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static bool hasUsesOutsideLoop(Instruction *I, Loop *L)
This node represents multiplication of some number of SCEVs.
static const SCEVConstant * getIncrmentFactorSCEV(ScalarEvolution *SE, const SCEV *SCEVExpr, Instruction &IV)
bool mayReadFromMemory() const
Return true if this instruction may read memory.
This node represents a polynomial recurrence on the trip count of the specified loop.
bool isAssociative() const
Return true if the instruction is associative:
Function Alias Analysis false
static bool add(uint64_t *dest, const uint64_t *x, const uint64_t *y, unsigned len)
This function adds the integer array x to the integer array Y and places the result in dest...
bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr)
Examine each PHI in the given block and delete it if it is dead.
const SCEV * getSizeOfExpr(Type *IntTy, Type *AllocTy)
Return an expression for sizeof AllocTy that is type IntTy.
An instruction for storing to memory.
Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Class to represent pointers.
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
initializer< Ty > init(const Ty &Val)
friend const_iterator end(StringRef path)
Get end iterator over path.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
Type * getType() const
Return the LLVM type of this SCEV expression.
Conditional or Unconditional Branch instruction.
This is an important base class in LLVM.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
int64_t getSExtValue() const
Get sign extended value.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< iterator, bool > insert(const ValueT &V)
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
This instruction compares its operands according to the predicate given to the constructor.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
op_range operands() const
Value * getOperand(unsigned i) const
static bool isLoopIncrement(User *U, Instruction *IV)
bool isCommutative() const
Return true if the instruction is commutative:
bool isPointerTy() const
True if this is an instance of PointerType.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
BinaryOps getOpcode() const
This is the common base class for memset/memcpy/memmove.
Iterator for intrusive lists based on ilist_node.
This is the shared class of boolean and integer constants.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static cl::opt< unsigned > NumToleratedFailedMatches("reroll-num-tolerated-failed-matches", cl::init(400), cl::Hidden, cl::desc("The maximum number of failures to tolerate"" during fuzzy matching. (default: 400)"))
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
ConstantInt * getValue() const
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Class for arbitrary precision integers.
bool isIntegerTy() const
True if this is an instance of IntegerType.
iterator_range< user_iterator > users()
This class uses information about analyze scalars to rewrite expressions in canonical form...
static bool isSimpleArithmeticOp(User *IVU)
Return true if IVU is a "simple" arithmetic operation.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
This class represents an analyzed expression in the program.
unsigned getNumBlocks() const
Get the number of blocks in this loop in constant time.
Represents a single loop in the control flow graph.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool hasOneUse() const
Return true if there is exactly one user of this value.
APFloat abs(APFloat X)
Returns the absolute value of the argument.
void getLoopAnalysisUsage(AnalysisUsage &AU)
Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.
iterator find(const KeyT &Val)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
LLVM Value Representation.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const SCEV * getUDivExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
bool isSameOperationAs(const Instruction *I, unsigned flags=0) const
This function determines if the specified instruction executes the same operation as the current one...
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
APInt abs() const
Get the absolute value;.
unsigned getNumUses() const
This method computes the number of uses of this Value.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
const BasicBlock * getParent() const
A wrapper class for inspecting calls to intrinsic functions.
This class represents a constant integer value.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.