38 #define DEBUG_TYPE "ifcvt"
61 STATISTIC(NumSimple,
"Number of simple if-conversions performed");
62 STATISTIC(NumSimpleFalse,
"Number of simple (F) if-conversions performed");
63 STATISTIC(NumTriangle,
"Number of triangle if-conversions performed");
64 STATISTIC(NumTriangleRev,
"Number of triangle (R) if-conversions performed");
65 STATISTIC(NumTriangleFalse,
"Number of triangle (F) if-conversions performed");
66 STATISTIC(NumTriangleFRev,
"Number of triangle (F/R) if-conversions performed");
67 STATISTIC(NumDiamonds,
"Number of diamond if-conversions performed");
68 STATISTIC(NumIfConvBBs,
"Number of if-converted blocks");
69 STATISTIC(NumDupBBs,
"Number of duplicated blocks");
70 STATISTIC(NumUnpred,
"Number of true blocks of diamonds unpredicated");
110 bool IsBeingAnalyzed : 1;
113 bool IsBrAnalyzable : 1;
114 bool HasFallThrough : 1;
115 bool IsUnpredicable : 1;
116 bool CannotBeCopied : 1;
117 bool ClobbersPred : 1;
118 unsigned NonPredSize;
126 BBInfo() : IsDone(
false), IsBeingAnalyzed(
false),
129 CannotBeCopied(
false), ClobbersPred(
false), NonPredSize(0),
130 ExtraCost(0), ExtraCost2(0), BB(nullptr), TrueBB(nullptr),
148 bool NeedSubsumption;
151 IfcvtToken(BBInfo &b, IfcvtKind k,
bool s,
unsigned d,
unsigned d2 = 0)
152 : BBI(b),
Kind(k), NeedSubsumption(s), NumDups(d), NumDups2(d2) {}
157 std::vector<BBInfo> BBAnalysis;
173 std::function<bool(const Function &)> PredicateFtor;
191 bool ReverseBranchCondition(BBInfo &BBI);
192 bool ValidSimple(BBInfo &TrueBBI,
unsigned &Dups,
194 bool ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
195 bool FalseBranch,
unsigned &Dups,
197 bool ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
198 unsigned &Dups1,
unsigned &Dups2)
const;
199 void ScanInstructions(BBInfo &BBI);
202 bool isTriangle =
false,
bool RevBranch =
false);
203 void AnalyzeBlocks(
MachineFunction &MF, std::vector<IfcvtToken*> &Tokens);
205 void RemoveExtraEdges(BBInfo &BBI);
206 bool IfConvertSimple(BBInfo &BBI, IfcvtKind
Kind);
207 bool IfConvertTriangle(BBInfo &BBI, IfcvtKind
Kind);
208 bool IfConvertDiamond(BBInfo &BBI, IfcvtKind
Kind,
209 unsigned NumDups1,
unsigned NumDups2);
210 void PredicateBlock(BBInfo &BBI,
214 void CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
216 bool IgnoreBr =
false);
217 void MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI,
bool AddEdges =
true);
220 unsigned Cycle,
unsigned Extra,
227 unsigned TCycle,
unsigned TExtra,
229 unsigned FCycle,
unsigned FExtra,
231 return TCycle > 0 && FCycle > 0 &&
237 bool blockAlwaysFallThrough(BBInfo &BBI)
const {
238 return BBI.IsBrAnalyzable && BBI.TrueBB ==
nullptr;
242 static bool IfcvtTokenCmp(IfcvtToken *C1, IfcvtToken *C2) {
243 int Incr1 = (C1->Kind == ICDiamond)
244 ? -(
int)(C1->NumDups + C1->NumDups2) : (
int)C1->NumDups;
245 int Incr2 = (C2->Kind == ICDiamond)
246 ? -(
int)(C2->NumDups + C2->NumDups2) : (
int)C2->NumDups;
249 else if (Incr1 == Incr2) {
251 if (!C1->NeedSubsumption && C2->NeedSubsumption)
253 else if (C1->NeedSubsumption == C2->NeedSubsumption) {
255 if ((
unsigned)C1->Kind < (
unsigned)C2->Kind)
257 else if (C1->Kind == C2->Kind)
258 return C1->BBI.BB->getNumber() < C2->BBI.BB->getNumber();
275 if (PredicateFtor && !PredicateFtor(*MF.getFunction()))
282 MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
283 MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
284 MRI = &MF.getRegInfo();
287 if (!
TII)
return false;
289 PreRegAlloc = MRI->isSSA();
291 bool BFChange =
false;
296 getAnalysisIfAvailable<MachineModuleInfo>());
299 DEBUG(
dbgs() <<
"\nIfcvt: function (" << ++FnNum <<
") \'"
300 << MF.getName() <<
"\'");
309 BBAnalysis.resize(MF.getNumBlockIDs());
311 std::vector<IfcvtToken*> Tokens;
313 unsigned NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle +
314 NumTriangleRev + NumTriangleFalse + NumTriangleFRev + NumDiamonds;
319 AnalyzeBlocks(MF, Tokens);
320 while (!Tokens.empty()) {
321 IfcvtToken *Token = Tokens.back();
323 BBInfo &BBI = Token->BBI;
324 IfcvtKind
Kind = Token->Kind;
325 unsigned NumDups = Token->NumDups;
326 unsigned NumDups2 = Token->NumDups2;
333 BBI.IsEnqueued =
false;
337 BBI.IsEnqueued =
false;
343 case ICSimpleFalse: {
344 bool isFalse = Kind == ICSimpleFalse;
346 DEBUG(
dbgs() <<
"Ifcvt (Simple" << (Kind == ICSimpleFalse ?
348 <<
"): BB#" << BBI.BB->getNumber() <<
" ("
349 << ((Kind == ICSimpleFalse)
350 ? BBI.FalseBB->getNumber()
351 : BBI.TrueBB->getNumber()) <<
") ");
352 RetVal = IfConvertSimple(BBI, Kind);
353 DEBUG(
dbgs() << (RetVal ?
"succeeded!" :
"failed!") <<
"\n");
355 if (isFalse) ++NumSimpleFalse;
362 case ICTriangleFalse:
363 case ICTriangleFRev: {
364 bool isFalse = Kind == ICTriangleFalse;
365 bool isRev = (Kind == ICTriangleRev || Kind == ICTriangleFRev);
375 DEBUG(
dbgs() <<
"): BB#" << BBI.BB->getNumber() <<
" (T:"
376 << BBI.TrueBB->getNumber() <<
",F:"
377 << BBI.FalseBB->getNumber() <<
") ");
378 RetVal = IfConvertTriangle(BBI, Kind);
379 DEBUG(
dbgs() << (RetVal ?
"succeeded!" :
"failed!") <<
"\n");
382 if (isRev) ++NumTriangleFRev;
383 else ++NumTriangleFalse;
385 if (isRev) ++NumTriangleRev;
393 DEBUG(
dbgs() <<
"Ifcvt (Diamond): BB#" << BBI.BB->getNumber() <<
" (T:"
394 << BBI.TrueBB->getNumber() <<
",F:"
395 << BBI.FalseBB->getNumber() <<
") ");
396 RetVal = IfConvertDiamond(BBI, Kind, NumDups, NumDups2);
397 DEBUG(
dbgs() << (RetVal ?
"succeeded!" :
"failed!") <<
"\n");
398 if (RetVal) ++NumDiamonds;
405 NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + NumTriangleRev +
406 NumTriangleFalse + NumTriangleFRev + NumDiamonds;
413 MadeChange |= Change;
417 while (!Tokens.empty()) {
418 IfcvtToken *Token = Tokens.back();
429 getAnalysisIfAvailable<MachineModuleInfo>());
432 MadeChange |= BFChange;
443 if (SuccBB != TrueBB)
451 bool IfConverter::ReverseBranchCondition(BBInfo &BBI) {
476 bool IfConverter::ValidSimple(BBInfo &TrueBBI,
unsigned &Dups,
479 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
482 if (TrueBBI.IsBrAnalyzable)
485 if (TrueBBI.BB->pred_size() > 1) {
486 if (TrueBBI.CannotBeCopied ||
490 Dups = TrueBBI.NonPredSize;
502 bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
503 bool FalseBranch,
unsigned &Dups,
506 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
509 if (TrueBBI.BB->pred_size() > 1) {
510 if (TrueBBI.CannotBeCopied)
513 unsigned Size = TrueBBI.NonPredSize;
514 if (TrueBBI.IsBrAnalyzable) {
515 if (TrueBBI.TrueBB && TrueBBI.BrCond.empty())
520 ? TrueBBI.TrueBB : TrueBBI.FalseBB;
532 if (!TExit && blockAlwaysFallThrough(TrueBBI)) {
534 if (++I == TrueBBI.BB->getParent()->end())
538 return TExit && TExit == FalseBBI.BB;
543 bool IfConverter::ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI,
544 unsigned &Dups1,
unsigned &Dups2)
const {
546 if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone ||
547 FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone)
553 if (!TT && blockAlwaysFallThrough(TrueBBI))
555 if (!FT && blockAlwaysFallThrough(FalseBBI))
559 if (!TT && (TrueBBI.IsBrAnalyzable || FalseBBI.IsBrAnalyzable))
561 if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1)
565 if (TrueBBI.FalseBB || FalseBBI.FalseBB ||
566 (TrueBBI.ClobbersPred && FalseBBI.ClobbersPred))
574 while (TIB != TIE && FIB != FIE) {
576 if (TIB->isDebugValue()) {
577 while (TIB != TIE && TIB->isDebugValue())
582 if (FIB->isDebugValue()) {
583 while (FIB != FIE && FIB->isDebugValue())
588 if (!TIB->isIdenticalTo(FIB))
599 if (!TIE->isBranch())
604 if (!FIE->isBranch())
610 if (TIB == TIE || FIB == FIE)
614 while (TIE != TIB && FIE != FIB) {
616 if (TIE->isDebugValue()) {
617 while (TIE != TIB && TIE->isDebugValue())
622 if (FIE->isDebugValue()) {
623 while (FIE != FIB && FIE->isDebugValue())
628 if (!TIE->isIdenticalTo(FIE))
643 void IfConverter::ScanInstructions(BBInfo &BBI) {
647 bool AlreadyPredicated = !BBI.Predicate.empty();
649 BBI.TrueBB = BBI.FalseBB =
nullptr;
653 BBI.HasFallThrough = BBI.IsBrAnalyzable && BBI.FalseBB ==
nullptr;
655 if (BBI.BrCond.size()) {
662 BBI.IsUnpredicable =
true;
671 BBI.ClobbersPred =
false;
674 if (I->isDebugValue())
677 if (I->isNotDuplicable())
678 BBI.CannotBeCopied =
true;
681 bool isCondBr = BBI.IsBrAnalyzable && I->isConditionalBranch();
689 unsigned ExtraPredCost =
TII->getPredicationCost(&*I);
690 unsigned NumCycles = SchedModel.computeInstrLatency(&*I,
false);
692 BBI.ExtraCost += NumCycles-1;
693 BBI.ExtraCost2 += ExtraPredCost;
694 }
else if (!AlreadyPredicated) {
698 BBI.IsUnpredicable =
true;
707 BBI.IsUnpredicable =
true;
713 std::vector<MachineOperand> PredDefs;
715 BBI.ClobbersPred =
true;
718 BBI.IsUnpredicable =
true;
726 bool IfConverter::FeasibilityAnalysis(BBInfo &BBI,
728 bool isTriangle,
bool RevBranch) {
730 if (BBI.IsDone || BBI.IsUnpredicable)
736 if (BBI.Predicate.size() && !BBI.IsBrAnalyzable)
744 if (BBI.BrCond.size()) {
767 std::vector<IfcvtToken*> &Tokens) {
779 while (!BBStack.empty()) {
780 BBState &State = BBStack.back();
782 BBInfo &BBI = BBAnalysis[BB->
getNumber()];
784 if (!State.SuccsAnalyzed) {
785 if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed) {
791 BBI.IsBeingAnalyzed =
true;
793 ScanInstructions(BBI);
797 if (!BBI.IsBrAnalyzable || BBI.BrCond.empty() || BBI.IsDone) {
798 BBI.IsBeingAnalyzed =
false;
799 BBI.IsAnalyzed =
true;
805 if (BBI.TrueBB == BB || BBI.FalseBB == BB) {
806 BBI.IsBeingAnalyzed =
false;
807 BBI.IsAnalyzed =
true;
814 BBI.IsBeingAnalyzed =
false;
815 BBI.IsAnalyzed =
true;
821 State.SuccsAnalyzed =
true;
822 BBStack.push_back(BBI.FalseBB);
823 BBStack.push_back(BBI.TrueBB);
827 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
828 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
830 if (TrueBBI.IsDone && FalseBBI.IsDone) {
831 BBI.IsBeingAnalyzed =
false;
832 BBI.IsAnalyzed =
true;
838 RevCond(BBI.BrCond.begin(), BBI.BrCond.end());
843 bool TNeedSub = !TrueBBI.Predicate.empty();
844 bool FNeedSub = !FalseBBI.Predicate.empty();
845 bool Enqueued =
false;
849 if (CanRevCond && ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2) &&
850 MeetIfcvtSizeLimit(*TrueBBI.BB, (TrueBBI.NonPredSize - (Dups + Dups2) +
851 TrueBBI.ExtraCost), TrueBBI.ExtraCost2,
852 *FalseBBI.BB, (FalseBBI.NonPredSize - (Dups + Dups2) +
853 FalseBBI.ExtraCost),FalseBBI.ExtraCost2,
855 FeasibilityAnalysis(TrueBBI, BBI.BrCond) &&
856 FeasibilityAnalysis(FalseBBI, RevCond)) {
865 Tokens.push_back(
new IfcvtToken(BBI, ICDiamond, TNeedSub|FNeedSub, Dups,
870 if (ValidTriangle(TrueBBI, FalseBBI,
false, Dups, Prediction) &&
871 MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
872 TrueBBI.ExtraCost2, Prediction) &&
873 FeasibilityAnalysis(TrueBBI, BBI.BrCond,
true)) {
881 Tokens.push_back(
new IfcvtToken(BBI, ICTriangle, TNeedSub, Dups));
885 if (ValidTriangle(TrueBBI, FalseBBI,
true, Dups, Prediction) &&
886 MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
887 TrueBBI.ExtraCost2, Prediction) &&
888 FeasibilityAnalysis(TrueBBI, BBI.BrCond,
true,
true)) {
889 Tokens.push_back(
new IfcvtToken(BBI, ICTriangleRev, TNeedSub, Dups));
893 if (ValidSimple(TrueBBI, Dups, Prediction) &&
894 MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost,
895 TrueBBI.ExtraCost2, Prediction) &&
896 FeasibilityAnalysis(TrueBBI, BBI.BrCond)) {
904 Tokens.push_back(
new IfcvtToken(BBI, ICSimple, TNeedSub, Dups));
910 if (ValidTriangle(FalseBBI, TrueBBI,
false, Dups,
912 MeetIfcvtSizeLimit(*FalseBBI.BB,
913 FalseBBI.NonPredSize + FalseBBI.ExtraCost,
914 FalseBBI.ExtraCost2, Prediction.
getCompl()) &&
915 FeasibilityAnalysis(FalseBBI, RevCond,
true)) {
916 Tokens.push_back(
new IfcvtToken(BBI, ICTriangleFalse, FNeedSub, Dups));
920 if (ValidTriangle(FalseBBI, TrueBBI,
true, Dups,
922 MeetIfcvtSizeLimit(*FalseBBI.BB,
923 FalseBBI.NonPredSize + FalseBBI.ExtraCost,
924 FalseBBI.ExtraCost2, Prediction.
getCompl()) &&
925 FeasibilityAnalysis(FalseBBI, RevCond,
true,
true)) {
926 Tokens.push_back(
new IfcvtToken(BBI, ICTriangleFRev, FNeedSub, Dups));
930 if (ValidSimple(FalseBBI, Dups, Prediction.
getCompl()) &&
931 MeetIfcvtSizeLimit(*FalseBBI.BB,
932 FalseBBI.NonPredSize + FalseBBI.ExtraCost,
933 FalseBBI.ExtraCost2, Prediction.
getCompl()) &&
934 FeasibilityAnalysis(FalseBBI, RevCond)) {
935 Tokens.push_back(
new IfcvtToken(BBI, ICSimpleFalse, FNeedSub, Dups));
940 BBI.IsEnqueued = Enqueued;
941 BBI.IsBeingAnalyzed =
false;
942 BBI.IsAnalyzed =
true;
950 std::vector<IfcvtToken*> &Tokens) {
953 AnalyzeBlock(BB, Tokens);
957 std::stable_sort(Tokens.begin(), Tokens.end(), IfcvtTokenCmp);
971 if (I == E || !I->empty() || !PI->isSuccessor(I))
983 BBInfo &PBBI = BBAnalysis[Predecessor->getNumber()];
984 if (PBBI.IsDone || PBBI.BB == BB)
986 PBBI.IsAnalyzed =
false;
987 PBBI.IsEnqueued =
false;
1002 void IfConverter::RemoveExtraEdges(BBInfo &BBI) {
1006 BBI.BB->CorrectExtraCFGEdges(TBB, FBB, !Cond.
empty());
1016 for (
auto Reg : Clobbers) {
1035 assert(Op.
isReg() &&
"Register operand required");
1051 if (!O->isReg() || !O->isKill())
1053 if (DontKill.
contains(O->getReg()))
1054 O->setIsKill(
false);
1066 for ( ; I != E; ++
I)
1072 bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind
Kind) {
1073 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1074 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1075 BBInfo *CvtBBI = &TrueBBI;
1076 BBInfo *NextBBI = &FalseBBI;
1079 if (Kind == ICSimpleFalse)
1082 if (CvtBBI->IsDone ||
1083 (CvtBBI->CannotBeCopied && CvtBBI->BB->pred_size() > 1)) {
1085 BBI.IsAnalyzed =
false;
1086 CvtBBI->IsAnalyzed =
false;
1090 if (CvtBBI->BB->hasAddressTaken())
1094 if (Kind == ICSimpleFalse)
1101 Redefs.addLiveIns(CvtBBI->BB);
1102 Redefs.addLiveIns(NextBBI->BB);
1107 DontKill.addLiveIns(NextBBI->BB);
1109 if (CvtBBI->BB->pred_size() > 1) {
1113 CopyAndPredicateBlock(BBI, *CvtBBI, Cond);
1117 BBI.BB->removeSuccessor(CvtBBI->BB);
1119 RemoveKills(CvtBBI->BB->begin(), CvtBBI->BB->end(), DontKill, *TRI);
1120 PredicateBlock(*CvtBBI, CvtBBI->BB->end(), Cond);
1124 MergeBlocks(BBI, *CvtBBI);
1127 bool IterIfcvt =
true;
1130 BBI.HasFallThrough =
false;
1144 RemoveExtraEdges(BBI);
1149 InvalidatePreds(BBI.BB);
1150 CvtBBI->IsDone =
true;
1164 uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
1165 uint32_t Scale = (NewMax / UINT32_MAX) + 1;
1171 else if (*
SI == FalseBB)
1180 bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
1181 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1182 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1183 BBInfo *CvtBBI = &TrueBBI;
1184 BBInfo *NextBBI = &FalseBBI;
1188 if (Kind == ICTriangleFalse || Kind == ICTriangleFRev)
1191 if (CvtBBI->IsDone ||
1192 (CvtBBI->CannotBeCopied && CvtBBI->BB->pred_size() > 1)) {
1194 BBI.IsAnalyzed =
false;
1195 CvtBBI->IsAnalyzed =
false;
1199 if (CvtBBI->BB->hasAddressTaken())
1203 if (Kind == ICTriangleFalse || Kind == ICTriangleFRev)
1207 if (Kind == ICTriangleRev || Kind == ICTriangleFRev) {
1208 if (ReverseBranchCondition(*CvtBBI)) {
1212 E = CvtBBI->BB->pred_end(); PI != E; ++PI) {
1216 BBInfo &PBBI = BBAnalysis[PBB->
getNumber()];
1217 if (PBBI.IsEnqueued) {
1218 PBBI.IsAnalyzed =
false;
1219 PBBI.IsEnqueued =
false;
1228 Redefs.addLiveIns(CvtBBI->BB);
1229 Redefs.addLiveIns(NextBBI->BB);
1233 bool HasEarlyExit = CvtBBI->FalseBB !=
nullptr;
1234 uint64_t CvtNext = 0, CvtFalse = 0, BBNext = 0, BBCvt = 0, SumWeight = 0;
1235 uint32_t WeightScale = 0;
1239 CvtNext = MBPI->getEdgeWeight(CvtBBI->BB, NextBBI->BB);
1240 CvtFalse = MBPI->getEdgeWeight(CvtBBI->BB, CvtBBI->FalseBB);
1241 BBNext = MBPI->getEdgeWeight(BBI.BB, NextBBI->BB);
1242 BBCvt = MBPI->getEdgeWeight(BBI.BB, CvtBBI->BB);
1243 SumWeight = MBPI->getSumForBlock(CvtBBI->BB, WeightScale);
1246 if (CvtBBI->BB->pred_size() > 1) {
1250 CopyAndPredicateBlock(BBI, *CvtBBI, Cond,
true);
1254 BBI.BB->removeSuccessor(CvtBBI->BB);
1258 PredicateBlock(*CvtBBI, CvtBBI->BB->end(), Cond);
1262 MergeBlocks(BBI, *CvtBBI,
false);
1268 CvtBBI->BrCond.end());
1272 BBI.BB->addSuccessor(CvtBBI->FalseBB);
1280 uint64_t NewNext = BBNext * SumWeight + (BBCvt * CvtNext) / WeightScale;
1281 uint64_t NewFalse = (BBCvt * CvtFalse) / WeightScale;
1286 CvtBBI->FalseBB, MBPI);
1291 bool FalseBBDead =
false;
1292 bool IterIfcvt =
true;
1294 if (!isFallThrough) {
1298 if (!HasEarlyExit &&
1299 NextBBI->BB->pred_size() == 1 && !NextBBI->HasFallThrough &&
1300 !NextBBI->BB->hasAddressTaken()) {
1301 MergeBlocks(BBI, *NextBBI);
1305 BBI.HasFallThrough =
false;
1312 RemoveExtraEdges(BBI);
1317 InvalidatePreds(BBI.BB);
1318 CvtBBI->IsDone =
true;
1320 NextBBI->IsDone =
true;
1328 bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
1329 unsigned NumDups1,
unsigned NumDups2) {
1330 BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()];
1331 BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()];
1335 if (blockAlwaysFallThrough(TrueBBI))
1336 TailBB = FalseBBI.TrueBB;
1337 assert((TailBB || !TrueBBI.IsBrAnalyzable) &&
"Unexpected!");
1340 if (TrueBBI.IsDone || FalseBBI.IsDone ||
1341 TrueBBI.BB->pred_size() > 1 ||
1342 FalseBBI.BB->pred_size() > 1) {
1344 BBI.IsAnalyzed =
false;
1345 TrueBBI.IsAnalyzed =
false;
1346 FalseBBI.IsAnalyzed =
false;
1350 if (TrueBBI.BB->hasAddressTaken() || FalseBBI.BB->hasAddressTaken())
1357 BBInfo *BBI1 = &TrueBBI;
1358 BBInfo *BBI2 = &FalseBBI;
1366 bool DoSwap =
false;
1367 if (TrueBBI.ClobbersPred && !FalseBBI.ClobbersPred)
1369 else if (TrueBBI.ClobbersPred == FalseBBI.ClobbersPred) {
1370 if (TrueBBI.NonPredSize > FalseBBI.NonPredSize)
1384 Redefs.addLiveIns(BBI1->BB);
1390 BBI1->NonPredSize -= NumDups1;
1391 BBI2->NonPredSize -= NumDups1;
1395 for (
unsigned i = 0; i < NumDups1; ++DI1) {
1396 if (!DI1->isDebugValue())
1399 while (NumDups1 != 0) {
1401 if (!DI2->isDebugValue())
1411 DontKill.stepBackward(*I);
1417 Redefs.stepForward(*I, IgnoredClobbers);
1419 BBI.BB->splice(BBI.BB->end(), BBI1->BB, BBI1->BB->begin(), DI1);
1420 BBI2->BB->erase(BBI2->BB->begin(), DI2);
1424 DI1 = BBI1->BB->end();
1425 for (
unsigned i = 0; i != NumDups2; ) {
1428 assert (DI1 != BBI1->BB->begin());
1431 if (!DI1->isDebugValue())
1434 BBI1->BB->erase(DI1, BBI1->BB->end());
1438 RemoveKills(BBI1->BB->begin(), BBI1->BB->end(), DontKill, *TRI);
1442 DI2 = BBI2->BB->end();
1443 while (NumDups2 != 0) {
1446 assert (DI2 != BBI2->BB->begin());
1449 if (!DI2->isDebugValue())
1463 if (
TII->isProfitableToUnpredicate(*BBI1->BB, *BBI2->BB)) {
1465 if (FI->isDebugValue())
1468 for (
unsigned i = 0, e = FI->getNumOperands(); i != e; ++i) {
1477 }
else if (!RedefsByFalse.
count(Reg)) {
1482 ExtUses.
insert(*SubRegs);
1486 for (
unsigned i = 0, e = Defs.
size(); i != e; ++i) {
1487 unsigned Reg = Defs[i];
1488 if (!ExtUses.
count(Reg)) {
1491 RedefsByFalse.
insert(*SubRegs);
1498 PredicateBlock(*BBI1, BBI1->BB->end(), *Cond1, &RedefsByFalse);
1501 PredicateBlock(*BBI2, DI2, *Cond2);
1504 MergeBlocks(BBI, *BBI1, TailBB ==
nullptr);
1505 MergeBlocks(BBI, *BBI2, TailBB ==
nullptr);
1512 BBInfo &TailBBI = BBAnalysis[TailBB->
getNumber()];
1513 bool CanMergeTail = !TailBBI.HasFallThrough &&
1514 !TailBBI.BB->hasAddressTaken();
1517 unsigned NumPreds = TailBB->
pred_size();
1519 CanMergeTail =
false;
1520 else if (NumPreds == 1 && CanMergeTail) {
1522 if (*PI != BBI1->BB && *PI != BBI2->BB)
1523 CanMergeTail =
false;
1526 MergeBlocks(BBI, TailBBI);
1527 TailBBI.IsDone =
true;
1529 BBI.BB->addSuccessor(TailBB);
1531 BBI.HasFallThrough =
false;
1538 BBI.BB->removeSuccessor(BBI1->BB);
1539 BBI.BB->removeSuccessor(BBI2->BB);
1540 RemoveExtraEdges(BBI);
1543 BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone =
true;
1544 InvalidatePreds(BBI.BB);
1560 unsigned Reg = MO.
getReg();
1572 void IfConverter::PredicateBlock(BBInfo &BBI,
1576 bool AnyUnpred =
false;
1577 bool MaySpec = LaterRedefs !=
nullptr;
1593 dbgs() <<
"Unable to predicate " << *I <<
"!\n";
1603 BBI.Predicate.append(Cond.
begin(), Cond.
end());
1605 BBI.IsAnalyzed =
false;
1606 BBI.NonPredSize = 0;
1615 void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI,
1621 E = FromBBI.BB->end(); I != E; ++
I) {
1623 if (IgnoreBr && I->isBranch())
1627 ToBBI.BB->insert(ToBBI.BB->end(),
MI);
1628 ToBBI.NonPredSize++;
1629 unsigned ExtraPredCost =
TII->getPredicationCost(&*I);
1630 unsigned NumCycles = SchedModel.computeInstrLatency(&*I,
false);
1632 ToBBI.ExtraCost += NumCycles-1;
1633 ToBBI.ExtraCost2 += ExtraPredCost;
1638 dbgs() <<
"Unable to predicate " << *I <<
"!\n";
1649 if (!DontKill.empty())
1654 std::vector<MachineBasicBlock *> Succs(FromBBI.BB->succ_begin(),
1655 FromBBI.BB->succ_end());
1659 for (
unsigned i = 0, e = Succs.size(); i != e; ++i) {
1662 if (Succ == FallThrough)
1664 ToBBI.BB->addSuccessor(Succ);
1668 ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end());
1669 ToBBI.Predicate.append(Cond.
begin(), Cond.
end());
1671 ToBBI.ClobbersPred |= FromBBI.ClobbersPred;
1672 ToBBI.IsAnalyzed =
false;
1682 void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI,
bool AddEdges) {
1683 assert(!FromBBI.BB->hasAddressTaken() &&
1684 "Removing a BB whose address is taken!");
1686 ToBBI.BB->splice(ToBBI.BB->end(),
1687 FromBBI.BB, FromBBI.BB->begin(), FromBBI.BB->end());
1689 std::vector<MachineBasicBlock *> Succs(FromBBI.BB->succ_begin(),
1690 FromBBI.BB->succ_end());
1694 for (
unsigned i = 0, e = Succs.size(); i != e; ++i) {
1697 if (Succ == FallThrough)
1699 FromBBI.BB->removeSuccessor(Succ);
1700 if (AddEdges && !ToBBI.BB->isSuccessor(Succ))
1701 ToBBI.BB->addSuccessor(Succ);
1705 if (NBB && !FromBBI.BB->isSuccessor(NBB))
1706 FromBBI.BB->addSuccessor(NBB);
1708 ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end());
1709 FromBBI.Predicate.clear();
1711 ToBBI.NonPredSize += FromBBI.NonPredSize;
1712 ToBBI.ExtraCost += FromBBI.ExtraCost;
1713 ToBBI.ExtraCost2 += FromBBI.ExtraCost2;
1714 FromBBI.NonPredSize = 0;
1715 FromBBI.ExtraCost = 0;
1716 FromBBI.ExtraCost2 = 0;
1718 ToBBI.ClobbersPred |= FromBBI.ClobbersPred;
1719 ToBBI.HasFallThrough = FromBBI.HasFallThrough;
1720 ToBBI.IsAnalyzed =
false;
1721 FromBBI.IsAnalyzed =
false;
1726 return new IfConverter(Ftor);
void push_back(const T &Elt)
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
STATISTIC(NumFunctions,"Total number of functions")
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
static cl::opt< bool > DisableDiamond("disable-ifcvt-diamond", cl::init(false), cl::Hidden)
static cl::opt< bool > DisableTriangleFR("disable-ifcvt-triangle-false-rev", cl::init(false), cl::Hidden)
bool SubsumesPredicate(ArrayRef< MachineOperand > Pred1, ArrayRef< MachineOperand > Pred2) const override
MIBundleOperands - Iterate over all operands in a bundle of machine instructions. ...
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
void setIsDead(bool Val=true)
static cl::opt< bool > DisableSimple("disable-ifcvt-simple", cl::init(false), cl::Hidden)
bool PredicateInstruction(MachineInstr *MI, ArrayRef< MachineOperand > Cond) const override
static void UpdatePredRedefs(MachineInstr *MI, LivePhysRegs &Redefs)
Behaves like LiveRegUnits::StepForward() but also adds implicit uses to all values defined in MI whic...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Provide an instruction scheduling machine model to CodeGen passes.
const HexagonInstrInfo * TII
bool isPredicated(const MachineInstr *MI) const override
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static MachineBasicBlock * getNextBlock(MachineBasicBlock *BB)
getNextBlock - Returns the next block in the function blocks ordering.
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
void initializeIfConverterPass(PassRegistry &)
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Number of individual test Apply this number of consecutive mutations to each input If
const HexagonRegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
FunctionPass * createIfConverter(std::function< bool(const Function &)> Ftor)
std::vector< MachineBasicBlock * >::iterator pred_iterator
void stepForward(const MachineInstr &MI, SmallVectorImpl< std::pair< unsigned, const MachineOperand * >> &Clobbers)
Simulates liveness when stepping forward over an instruction(bundle): Remove killed-uses, add defs.
static void ScaleWeights(uint64_t NewTrue, uint64_t NewFalse, MachineBasicBlock *MBB, const MachineBasicBlock *TrueBB, const MachineBasicBlock *FalseBB, const MachineBranchProbabilityInfo *MBPI)
Scale down weights to fit into uint32_t.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
static cl::opt< bool > DisableTriangleR("disable-ifcvt-triangle-rev", cl::init(false), cl::Hidden)
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, const BranchProbability &Probability) const override
static MachineBasicBlock * findFalseBlock(MachineBasicBlock *BB, MachineBasicBlock *TrueBB)
findFalseBlock - BB has a fallthrough.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, unsigned ExtraPredCycles, const BranchProbability &Probability) const override
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned RemoveBranch(MachineBasicBlock &MBB) const override
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
bool ReverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static cl::opt< bool > DisableTriangleF("disable-ifcvt-triangle-false", cl::init(false), cl::Hidden)
Represent the analysis usage information of a pass.
static void InsertUncondBranch(MachineBasicBlock *BB, MachineBasicBlock *ToBB, const TargetInstrInfo *TII)
InsertUncondBranch - Inserts an unconditional branch from BB to ToBB.
bool contains(unsigned Reg) const
Returns true if register Reg is contained in the set.
FunctionPass class - This class is used to implement most global optimizations.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
iterator_range< pred_iterator > predecessors()
succ_iterator succ_begin()
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
pred_iterator pred_begin()
MCSubRegIterator enumerates all sub-registers of Reg.
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
static bool MaySpeculate(const MachineInstr *MI, SmallSet< unsigned, 4 > &LaterRedefs)
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
bool DefinesPredicate(MachineInstr *MI, std::vector< MachineOperand > &Pred) const override
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
CloneMachineInstr - Create a new MachineInstr which is a copy of the 'Orig' instruction, identical in all ways except the instruction has no parent, prev, or next.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static cl::opt< bool > DisableTriangle("disable-ifcvt-triangle", cl::init(false), cl::Hidden)
virtual const TargetLowering * getTargetLowering() const
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static void RemoveKills(MachineInstr &MI, const LivePhysRegs &DontKill)
Remove kill flags from operands with a registers in the DontKill set.
static cl::opt< int > IfCvtLimit("ifcvt-limit", cl::init(-1), cl::Hidden)
bool isPredicable(MachineInstr *MI) const override
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool isPredicated(MCInstrInfo const &MCII, MCInst const &MCI)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
Representation of each machine instruction.
bundle_iterator< const MachineInstr, const_instr_iterator > const_iterator
static bool canFallThroughTo(MachineBasicBlock *BB, MachineBasicBlock *ToBB)
canFallThroughTo - Returns true either if ToBB is the next block after BB or that all the intervening...
static cl::opt< int > IfCvtFnStart("ifcvt-fn-start", cl::init(-1), cl::Hidden)
A set of live physical registers with functions to track liveness when walking backward/forward throu...
static cl::opt< bool > IfCvtBranchFold("ifcvt-branch-fold", cl::init(true), cl::Hidden)
char & IfConverterID
IfConverter - This pass performs machine code if conversion.
static cl::opt< bool > DisableSimpleF("disable-ifcvt-simple-false", cl::init(false), cl::Hidden)
unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, DebugLoc DL) const override
virtual unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, DebugLoc DL) const
Insert branch code into the end of the specified MachineBasicBlock.
void setSuccWeight(succ_iterator I, uint32_t weight)
Set successor weight of a given iterator.
uint32_t getEdgeWeight(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
unsigned getReg() const
getReg - Returns the register number.
bool isValid() const
isValid - Returns true until all the operands have been visited.
const ARM::ArchExtKind Kind
virtual const TargetInstrInfo * getInstrInfo() const
std::reverse_iterator< iterator > reverse_iterator
BasicBlockListType::iterator iterator
print Print MemDeps of function
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool OptimizeFunction(MachineFunction &MF, const TargetInstrInfo *tii, const TargetRegisterInfo *tri, MachineModuleInfo *mmi)
OptimizeFunction - Perhaps branch folding, tail merging and other CFG optimizations on the given func...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
addReg - Add a new virtual register operand...
unsigned pred_size() const
This file describes how to lower LLVM code to machine code.
static cl::opt< int > IfCvtFnStop("ifcvt-fn-stop", cl::init(-1), cl::Hidden)
BranchProbability getCompl() const