104#define DEBUG_TYPE "peephole-opt"
108 cl::desc(
"Aggressive extension optimization"));
112 cl::desc(
"Disable the peephole optimizer"));
119 cl::desc(
"Disable advanced copy optimization"));
123 cl::desc(
"Disable non-allocatable physical register copy optimization"));
129 cl::desc(
"Limit the length of PHI chains to lookup"));
135 cl::desc(
"Maximum length of recurrence chain when evaluating the benefit "
136 "of commuting operands"));
138STATISTIC(NumReuse,
"Number of extension results reused");
140STATISTIC(NumImmFold,
"Number of move immediate folded");
143STATISTIC(NumUncoalescableCopies,
"Number of uncoalescable copies optimized");
144STATISTIC(NumRewrittenCopies,
"Number of copies rewritten");
145STATISTIC(NumNAPhysCopies,
"Number of non-allocatable physical copies removed");
149class ValueTrackerResult;
150class RecurrenceInstr;
156 unsigned CurrentSrcIdx = 0;
159 virtual ~Rewriter() =
default;
191 virtual bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg) = 0;
195class CopyRewriter :
public Rewriter {
198 assert(
MI.isCopy() &&
"Expected copy instruction");
200 virtual ~CopyRewriter() =
default;
205 if (CurrentSrcIdx > 0)
219 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
220 if (CurrentSrcIdx != 1)
231class UncoalescableRewriter :
public Rewriter {
236 NumDefs =
MI.getDesc().getNumDefs();
246 if (CurrentSrcIdx == NumDefs)
249 while (CopyLike.getOperand(CurrentSrcIdx).isDead()) {
251 if (CurrentSrcIdx == NumDefs)
264 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
270class InsertSubregRewriter :
public Rewriter {
273 assert(
MI.isInsertSubreg() &&
"Invalid instruction");
290 if (CurrentSrcIdx == 2)
304 (
unsigned)CopyLike.getOperand(3).getImm());
308 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
309 if (CurrentSrcIdx != 2)
320class ExtractSubregRewriter :
public Rewriter {
326 assert(
MI.isExtractSubreg() &&
"Invalid instruction");
337 if (CurrentSrcIdx == 1)
355 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
357 if (CurrentSrcIdx != 1)
360 CopyLike.getOperand(CurrentSrcIdx).setReg(NewReg);
371 CopyLike.removeOperand(2);
373 CopyLike.setDesc(
TII.get(TargetOpcode::COPY));
376 CopyLike.getOperand(CurrentSrcIdx + 1).setImm(NewSubReg);
382class RegSequenceRewriter :
public Rewriter {
385 assert(
MI.isRegSequence() &&
"Invalid instruction");
409 if (CurrentSrcIdx == 0) {
414 if (CurrentSrcIdx >= CopyLike.getNumOperands())
417 const MachineOperand &MOInsertedReg = CopyLike.getOperand(CurrentSrcIdx);
418 Src.Reg = MOInsertedReg.
getReg();
420 if ((Src.SubReg = MOInsertedReg.
getSubReg()))
425 Dst.SubReg = CopyLike.getOperand(CurrentSrcIdx + 1).getImm();
429 assert(MODef.
getSubReg() == 0 &&
"cannot have subregister def in SSA");
433 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
436 if ((CurrentSrcIdx & 1) != 1 || CurrentSrcIdx > CopyLike.getNumOperands())
455 : DT(DT), MLI(MLI) {}
472 bool optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter);
477 bool findNextSource(
RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap);
488 bool findTargetRecurrence(
Register Reg,
490 RecurrenceCycle &RC);
519 return MI.isCopy() ||
521 MI.isExtractSubreg()));
528 MI.isInsertSubregLike() ||
529 MI.isExtractSubregLike()));
533 RewriteMapTy &RewriteMap);
547 unsigned SrcSubReg =
MI.getOperand(1).getSubReg();
548 if (!SrcReg.
isVirtual() && !
MRI->isConstantPhysReg(SrcReg))
559 if (!getCopySrc(
MI, SrcPair))
562 auto It = CopySrcMIs.
find(SrcPair);
563 if (It != CopySrcMIs.
end() && It->second == &
MI)
564 CopySrcMIs.
erase(It);
570 deleteChangedCopy(
MI);
597 MachineFunctionProperties::Property::IsSSA);
607class RecurrenceInstr {
609 using IndexPair = std::pair<unsigned, unsigned>;
612 RecurrenceInstr(
MachineInstr *MI,
unsigned Idx1,
unsigned Idx2)
613 :
MI(
MI), CommutePair(
std::make_pair(Idx1, Idx2)) {}
616 std::optional<IndexPair> getCommutePair()
const {
return CommutePair; }
620 std::optional<IndexPair> CommutePair;
626class ValueTrackerResult {
635 ValueTrackerResult() =
default;
639 bool isValid()
const {
return getNumSources() > 0; }
649 void addSource(
Register SrcReg,
unsigned SrcSubReg) {
653 void setSource(
int Idx,
Register SrcReg,
unsigned SrcSubReg) {
654 assert(
Idx < getNumSources() &&
"Reg pair source out of index");
658 int getNumSources()
const {
return RegSrcs.
size(); }
663 assert(
Idx < getNumSources() &&
"Reg source out of index");
664 return RegSrcs[
Idx].Reg;
667 unsigned getSrcSubReg(
int Idx)
const {
668 assert(
Idx < getNumSources() &&
"SubReg source out of index");
669 return RegSrcs[
Idx].SubReg;
673 if (
Other.getInst() != getInst())
676 if (
Other.getNumSources() != getNumSources())
679 for (
int i = 0, e =
Other.getNumSources(); i != e; ++i)
680 if (
Other.getSrcReg(i) != getSrcReg(i) ||
681 Other.getSrcSubReg(i) != getSrcSubReg(i))
724 ValueTrackerResult getNextSourceImpl();
727 ValueTrackerResult getNextSourceFromCopy();
730 ValueTrackerResult getNextSourceFromBitcast();
733 ValueTrackerResult getNextSourceFromRegSequence();
736 ValueTrackerResult getNextSourceFromInsertSubreg();
739 ValueTrackerResult getNextSourceFromExtractSubreg();
742 ValueTrackerResult getNextSourceFromSubregToReg();
745 ValueTrackerResult getNextSourceFromPHI();
760 if (!
Reg.isPhysical()) {
761 Def =
MRI.getVRegDef(Reg);
762 DefIdx =
MRI.def_begin(Reg).getOperandNo();
771 ValueTrackerResult getNextSource();
776char PeepholeOptimizerLegacy::ID = 0;
781 "Peephole Optimizations",
false,
false)
795bool PeepholeOptimizer::optimizeExtInstr(
800 if (!
TII->isCoalescableExtInstr(
MI, SrcReg, DstReg, SubIdx))
806 if (
MRI->hasOneNonDBGUse(SrcReg))
813 DstRC =
TRI->getSubClassWithSubReg(DstRC, SubIdx);
823 TRI->getSubClassWithSubReg(
MRI->getRegClass(SrcReg), SubIdx) !=
nullptr;
829 ReachedBBs.
insert(UI.getParent());
837 bool ExtendLife =
true;
849 if (UseSrcSubIdx && UseMO.getSubReg() != SubIdx)
873 if (UseMBB == &
MBB) {
875 if (!LocalMIs.count(
UseMI))
876 Uses.push_back(&UseMO);
877 }
else if (ReachedBBs.
count(UseMBB)) {
880 Uses.push_back(&UseMO);
893 if (ExtendLife && !ExtendedUses.
empty())
898 bool Changed =
false;
907 PHIBBs.
insert(UI.getParent());
913 if (PHIBBs.
count(UseMBB))
918 MRI->clearKillFlags(DstReg);
919 MRI->constrainRegClass(DstReg, DstRC);
941 TII->get(TargetOpcode::COPY), NewVR)
942 .
addReg(DstReg, 0, SubIdx);
946 UseMO->setReg(NewVR);
963 int64_t CmpMask, CmpValue;
970 if (
TII->optimizeCompareInstr(
MI, SrcReg, SrcReg2, CmpMask, CmpValue,
MRI)) {
980bool PeepholeOptimizer::optimizeSelect(
983 unsigned FalseOp = 0;
984 bool Optimizable =
false;
986 if (
TII->analyzeSelect(
MI,
Cond, TrueOp, FalseOp, Optimizable))
990 if (!
TII->optimizeSelect(
MI, LocalMIs))
993 MI.eraseFromParent();
1000 return TII->optimizeCondBranch(
MI);
1014bool PeepholeOptimizer::findNextSource(
RegSubRegPair RegSubReg,
1015 RewriteMapTy &RewriteMap) {
1021 if (
Reg.isPhysical())
1029 unsigned PHICount = 0;
1036 ValueTracker ValTracker(CurSrcPair.
Reg, CurSrcPair.
SubReg, *
MRI,
TII);
1041 ValueTrackerResult Res = ValTracker.getNextSource();
1047 ValueTrackerResult CurSrcRes = RewriteMap.lookup(CurSrcPair);
1048 if (CurSrcRes.isValid()) {
1049 assert(CurSrcRes == Res &&
"ValueTrackerResult found must match");
1052 if (CurSrcRes.getNumSources() > 1) {
1054 <<
"findNextSource: found PHI cycle, aborting...\n");
1059 RewriteMap.insert(std::make_pair(CurSrcPair, Res));
1063 unsigned NumSrcs = Res.getNumSources();
1071 for (
unsigned i = 0; i < NumSrcs; ++i)
1076 CurSrcPair = Res.getSrc(0);
1086 if (!
TRI->shouldRewriteCopySrc(DefRC, RegSubReg.
SubReg, SrcRC,
1092 if (PHICount > 0 && CurSrcPair.
SubReg != 0)
1098 }
while (!SrcToLook.
empty());
1101 return CurSrcPair.
Reg !=
Reg;
1113 assert(!SrcRegs.
empty() &&
"No sources to create a PHI instruction?");
1118 assert(SrcRegs[0].
SubReg == 0 &&
"should not have subreg operand");
1119 Register NewVR =
MRI.createVirtualRegister(NewRC);
1122 TII.get(TargetOpcode::PHI), NewVR);
1124 unsigned MBBOpIdx = 2;
1126 MIB.
addReg(RegPair.Reg, 0, RegPair.SubReg);
1131 MRI.clearKillFlags(RegPair.Reg);
1148 bool HandleMultipleSources =
true) {
1151 ValueTrackerResult Res = RewriteMap.
lookup(LookupSrc);
1157 unsigned NumSrcs = Res.getNumSources();
1159 LookupSrc.
Reg = Res.getSrcReg(0);
1160 LookupSrc.
SubReg = Res.getSrcSubReg(0);
1165 if (!HandleMultipleSources)
1171 for (
unsigned i = 0; i < NumSrcs; ++i) {
1172 RegSubRegPair PHISrc(Res.getSrcReg(i), Res.getSrcSubReg(i));
1190bool PeepholeOptimizer::optimizeCoalescableCopyImpl(Rewriter &&CpyRewriter) {
1191 bool Changed =
false;
1196 while (CpyRewriter.getNextRewritableSource(Src, TrackPair)) {
1198 RewriteMapTy RewriteMap;
1201 if (!findNextSource(TrackPair, RewriteMap))
1208 if (Src.Reg == NewSrc.
Reg || NewSrc.
Reg == 0)
1212 if (CpyRewriter.RewriteCurrentSource(NewSrc.
Reg, NewSrc.
SubReg)) {
1214 MRI->clearKillFlags(NewSrc.
Reg);
1224 NumRewrittenCopies += Changed;
1239bool PeepholeOptimizer::optimizeCoalescableCopy(
MachineInstr &
MI) {
1240 assert(isCoalescableCopy(
MI) &&
"Invalid argument");
1241 assert(
MI.getDesc().getNumDefs() == 1 &&
1242 "Coalescer can understand multiple defs?!");
1248 switch (
MI.getOpcode()) {
1249 case TargetOpcode::COPY:
1250 return optimizeCoalescableCopyImpl(CopyRewriter(
MI));
1251 case TargetOpcode::INSERT_SUBREG:
1252 return optimizeCoalescableCopyImpl(InsertSubregRewriter(
MI));
1253 case TargetOpcode::EXTRACT_SUBREG:
1254 return optimizeCoalescableCopyImpl(ExtractSubregRewriter(
MI, *
TII));
1255 case TargetOpcode::REG_SEQUENCE:
1256 return optimizeCoalescableCopyImpl(RegSequenceRewriter(
MI));
1259 if (
MI.isBitcast() ||
MI.isRegSequenceLike() ||
MI.isInsertSubregLike() ||
1260 MI.isExtractSubregLike())
1261 return optimizeCoalescableCopyImpl(UncoalescableRewriter(
MI));
1273 RewriteMapTy &RewriteMap) {
1274 assert(!
Def.Reg.isPhysical() &&
"We do not rewrite physical registers");
1281 Register NewVReg =
MRI->createVirtualRegister(DefRC);
1285 TII->get(TargetOpcode::COPY), NewVReg)
1296 MRI->replaceRegWith(
Def.Reg, NewVReg);
1297 MRI->clearKillFlags(NewVReg);
1301 MRI->clearKillFlags(NewSrc.
Reg);
1317bool PeepholeOptimizer::optimizeUncoalescableCopy(
1319 assert(isUncoalescableCopy(
MI) &&
"Invalid argument");
1320 UncoalescableRewriter CpyRewriter(
MI);
1325 RewriteMapTy RewriteMap;
1329 while (CpyRewriter.getNextRewritableSource(Src, Def)) {
1332 if (
Def.Reg.isPhysical())
1337 if (!findNextSource(Def, RewriteMap))
1347 LocalMIs.
insert(&NewCopy);
1352 MI.eraseFromParent();
1353 ++NumUncoalescableCopies;
1360bool PeepholeOptimizer::isLoadFoldable(
1362 if (!
MI.canFoldAsLoad() || !
MI.mayLoad())
1372 if (
Reg.isVirtual() && !
MI.getOperand(0).getSubReg() &&
1373 MRI->hasOneNonDBGUser(Reg)) {
1374 FoldAsLoadDefCandidates.
insert(Reg);
1380bool PeepholeOptimizer::isMoveImmediate(
1384 if (MCID.
getNumDefs() != 1 || !
MI.getOperand(0).isReg())
1387 if (!
Reg.isVirtual())
1391 if (!
MI.isMoveImmediate() && !
TII->getConstValDefinedInReg(
MI, Reg, ImmVal))
1394 ImmDefMIs.
insert(std::make_pair(Reg, &
MI));
1402bool PeepholeOptimizer::foldImmediate(
1406 for (
unsigned i = 0, e =
MI.getDesc().getNumOperands(); i != e; ++i) {
1411 if (!
Reg.isVirtual())
1413 if (ImmDefRegs.
count(Reg) == 0)
1416 assert(
II != ImmDefMIs.
end() &&
"couldn't find immediate definition");
1417 if (
TII->foldImmediate(
MI, *
II->second, Reg,
MRI)) {
1422 if (
MRI->getVRegDef(Reg) &&
1426 MRI->getRegClass(DstReg) ==
MRI->getRegClass(Reg)) {
1427 MRI->replaceRegWith(DstReg, Reg);
1428 MI.eraseFromParent();
1453 assert(
MI.isCopy() &&
"expected a COPY machine instruction");
1456 if (!getCopySrc(
MI, SrcPair))
1463 if (CopySrcMIs.
insert(std::make_pair(SrcPair, &
MI)).second) {
1471 "Unexpected mismatching subreg!");
1479 if (
MRI->getRegClass(DstReg) !=
MRI->getRegClass(PrevDstReg))
1482 MRI->replaceRegWith(DstReg, PrevDstReg);
1485 MRI->clearKillFlags(PrevDstReg);
1489bool PeepholeOptimizer::isNAPhysCopy(
Register Reg) {
1490 return Reg.isPhysical() && !
MRI->isAllocatable(Reg);
1493bool PeepholeOptimizer::foldRedundantNAPhysCopy(
1495 assert(
MI.isCopy() &&
"expected a COPY machine instruction");
1502 if (isNAPhysCopy(SrcReg) && DstReg.
isVirtual()) {
1506 NAPhysToVirtMIs.
insert({SrcReg, &
MI});
1510 if (!(SrcReg.
isVirtual() && isNAPhysCopy(DstReg)))
1514 auto PrevCopy = NAPhysToVirtMIs.
find(DstReg);
1515 if (PrevCopy == NAPhysToVirtMIs.
end()) {
1518 LLVM_DEBUG(
dbgs() <<
"NAPhysCopy: intervening clobber forbids erasing "
1524 if (PrevDstReg == SrcReg) {
1537 NAPhysToVirtMIs.
erase(PrevCopy);
1546bool PeepholeOptimizer::findTargetRecurrence(
1548 RecurrenceCycle &RC) {
1550 if (TargetRegs.
count(Reg))
1558 if (!
MRI->hasOneNonDBGUse(Reg))
1566 unsigned Idx =
MI.findRegisterUseOperandIdx(Reg,
nullptr);
1570 if (
MI.getDesc().getNumDefs() != 1)
1580 unsigned TiedUseIdx;
1581 if (!
MI.isRegTiedToUseOperand(0, &TiedUseIdx))
1584 if (
Idx == TiedUseIdx) {
1585 RC.push_back(RecurrenceInstr(&
MI));
1586 return findTargetRecurrence(DefOp.
getReg(), TargetRegs, RC);
1590 if (
TII->findCommutedOpIndices(
MI,
Idx, CommIdx) && CommIdx == TiedUseIdx) {
1591 RC.push_back(RecurrenceInstr(&
MI,
Idx, CommIdx));
1592 return findTargetRecurrence(DefOp.
getReg(), TargetRegs, RC);
1619 for (
unsigned Idx = 1;
Idx <
PHI.getNumOperands();
Idx += 2) {
1625 bool Changed =
false;
1627 if (findTargetRecurrence(
PHI.getOperand(0).getReg(), TargetRegs, RC)) {
1631 for (
auto &RI : RC) {
1633 auto CP = RI.getCommutePair();
1636 TII->commuteInstruction(*(RI.getMI()),
false, (*CP).first,
1653 PeepholeOptimizer Impl(DT, MLI);
1654 bool Changed = Impl.run(MF);
1665bool PeepholeOptimizerLegacy::runOnMachineFunction(
MachineFunction &MF) {
1669 ? &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree()
1671 auto *MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
1672 PeepholeOptimizer Impl(DT, MLI);
1673 return Impl.run(MF);
1678 LLVM_DEBUG(
dbgs() <<
"********** PEEPHOLE OPTIMIZER **********\n");
1689 bool Changed =
false;
1692 bool SeenMoveImm =
false;
1725 if (
MI->isDebugInstr())
1728 if (
MI->isPosition())
1731 if (IsLoopHeader &&
MI->isPHI()) {
1732 if (optimizeRecurrence(*
MI)) {
1738 if (!
MI->isCopy()) {
1743 if (MO.
isDef() && isNAPhysCopy(Reg)) {
1744 const auto &
Def = NAPhysToVirtMIs.
find(Reg);
1745 if (Def != NAPhysToVirtMIs.
end()) {
1749 <<
"NAPhysCopy: invalidating because of " << *
MI);
1750 NAPhysToVirtMIs.
erase(Def);
1755 for (
auto &RegMI : NAPhysToVirtMIs) {
1759 <<
"NAPhysCopy: invalidating because of " << *
MI);
1760 NAPhysToVirtMIs.erase(Def);
1767 if (
MI->isImplicitDef() ||
MI->isKill())
1770 if (
MI->isInlineAsm() ||
MI->hasUnmodeledSideEffects()) {
1777 NAPhysToVirtMIs.clear();
1780 if ((isUncoalescableCopy(*
MI) &&
1781 optimizeUncoalescableCopy(*
MI, LocalMIs)) ||
1782 (
MI->isCompare() && optimizeCmpInstr(*
MI)) ||
1783 (
MI->isSelect() && optimizeSelect(*
MI, LocalMIs))) {
1790 if (
MI->isConditionalBranch() && optimizeCondBranch(*
MI)) {
1795 if (isCoalescableCopy(*
MI) && optimizeCoalescableCopy(*
MI)) {
1801 if (
MI->isCopy() && (foldRedundantCopy(*
MI) ||
1802 foldRedundantNAPhysCopy(*
MI, NAPhysToVirtMIs))) {
1805 MI->eraseFromParent();
1810 if (isMoveImmediate(*
MI, ImmDefRegs, ImmDefMIs)) {
1813 Changed |= optimizeExtInstr(*
MI,
MBB, LocalMIs);
1821 Changed |= foldImmediate(*
MI, ImmDefRegs, ImmDefMIs,
Deleted);
1832 if (!isLoadFoldable(*
MI, FoldAsLoadDefCandidates) &&
1833 !FoldAsLoadDefCandidates.
empty()) {
1841 for (
unsigned i = MIDesc.
getNumDefs(); i !=
MI->getNumOperands(); ++i) {
1846 if (FoldAsLoadDefCandidates.
count(FoldAsLoadDefReg)) {
1851 Register FoldedReg = FoldAsLoadDefReg;
1854 TII->optimizeLoadInstr(*
MI,
MRI, FoldAsLoadDefReg,
DefMI)) {
1863 if (
MI->shouldUpdateAdditionalCallInfo())
1864 MI->getMF()->moveAdditionalCallInfo(
MI, FoldMI);
1865 MI->eraseFromParent();
1867 MRI->markUsesInDebugValueAsUndef(FoldedReg);
1868 FoldAsLoadDefCandidates.
erase(FoldedReg);
1882 if (
MI->isLoadFoldBarrier()) {
1884 FoldAsLoadDefCandidates.
clear();
1889 MF.resetDelegate(
this);
1893ValueTrackerResult ValueTracker::getNextSourceFromCopy() {
1894 assert(
Def->isCopy() &&
"Invalid definition");
1899 assert(
Def->getNumOperands() -
Def->getNumImplicitOperands() == 2 &&
1900 "Invalid number of operands");
1901 assert(!
Def->hasImplicitDef() &&
"Only implicit uses are allowed");
1903 if (
Def->getOperand(DefIdx).getSubReg() != DefSubReg)
1906 return ValueTrackerResult();
1910 return ValueTrackerResult();
1911 return ValueTrackerResult(Src.getReg(), Src.getSubReg());
1914ValueTrackerResult ValueTracker::getNextSourceFromBitcast() {
1915 assert(
Def->isBitcast() &&
"Invalid definition");
1918 if (
Def->mayRaiseFPException() ||
Def->hasUnmodeledSideEffects())
1919 return ValueTrackerResult();
1922 if (
Def->getDesc().getNumDefs() != 1)
1923 return ValueTrackerResult();
1928 return ValueTrackerResult();
1930 unsigned SrcIdx =
Def->getNumOperands();
1931 for (
unsigned OpIdx = DefIdx + 1, EndOpIdx = SrcIdx; OpIdx != EndOpIdx;
1939 assert(!MO.
isDef() &&
"We should have skipped all the definitions by now");
1940 if (SrcIdx != EndOpIdx)
1942 return ValueTrackerResult();
1948 if (SrcIdx >=
Def->getNumOperands())
1949 return ValueTrackerResult();
1954 if (
UseMI.isSubregToReg())
1955 return ValueTrackerResult();
1960 return ValueTrackerResult();
1961 return ValueTrackerResult(Src.getReg(), Src.getSubReg());
1964ValueTrackerResult ValueTracker::getNextSourceFromRegSequence() {
1965 assert((
Def->isRegSequence() ||
Def->isRegSequenceLike()) &&
1966 "Invalid definition");
1968 if (
Def->getOperand(DefIdx).getSubReg())
1983 return ValueTrackerResult();
1986 if (!
TII->getRegSequenceInputs(*Def, DefIdx, RegSeqInputRegs))
1987 return ValueTrackerResult();
1993 if (RegSeqInput.SubIdx == DefSubReg)
1994 return ValueTrackerResult(RegSeqInput.Reg, RegSeqInput.SubReg);
2000 return ValueTrackerResult();
2003ValueTrackerResult ValueTracker::getNextSourceFromInsertSubreg() {
2004 assert((
Def->isInsertSubreg() ||
Def->isInsertSubregLike()) &&
2005 "Invalid definition");
2007 if (
Def->getOperand(DefIdx).getSubReg())
2011 return ValueTrackerResult();
2015 if (!
TII->getInsertSubregInputs(*Def, DefIdx, BaseReg, InsertedReg))
2016 return ValueTrackerResult();
2025 if (InsertedReg.
SubIdx == DefSubReg) {
2026 return ValueTrackerResult(InsertedReg.
Reg, InsertedReg.
SubReg);
2035 if (
MRI.getRegClass(MODef.
getReg()) !=
MRI.getRegClass(BaseReg.
Reg) ||
2037 return ValueTrackerResult();
2042 if ((
TRI->getSubRegIndexLaneMask(DefSubReg) &
2043 TRI->getSubRegIndexLaneMask(InsertedReg.
SubIdx))
2045 return ValueTrackerResult();
2048 return ValueTrackerResult(BaseReg.
Reg, DefSubReg);
2051ValueTrackerResult ValueTracker::getNextSourceFromExtractSubreg() {
2052 assert((
Def->isExtractSubreg() ||
Def->isExtractSubregLike()) &&
2053 "Invalid definition");
2060 return ValueTrackerResult();
2063 if (!
TII->getExtractSubregInputs(*Def, DefIdx, ExtractSubregInputReg))
2064 return ValueTrackerResult();
2068 if (ExtractSubregInputReg.
SubReg)
2069 return ValueTrackerResult();
2071 return ValueTrackerResult(ExtractSubregInputReg.
Reg,
2072 ExtractSubregInputReg.
SubIdx);
2075ValueTrackerResult ValueTracker::getNextSourceFromSubregToReg() {
2076 assert(
Def->isSubregToReg() &&
"Invalid definition");
2084 if (DefSubReg !=
Def->getOperand(3).getImm())
2085 return ValueTrackerResult();
2088 if (
Def->getOperand(2).getSubReg())
2089 return ValueTrackerResult();
2091 return ValueTrackerResult(
Def->getOperand(2).getReg(),
2092 Def->getOperand(3).getImm());
2096ValueTrackerResult ValueTracker::getNextSourceFromPHI() {
2097 assert(
Def->isPHI() &&
"Invalid definition");
2098 ValueTrackerResult Res;
2102 if (
Def->getOperand(0).getSubReg() != DefSubReg)
2103 return ValueTrackerResult();
2106 for (
unsigned i = 1, e =
Def->getNumOperands(); i < e; i += 2) {
2112 return ValueTrackerResult();
2119ValueTrackerResult ValueTracker::getNextSourceImpl() {
2120 assert(Def &&
"This method needs a valid definition");
2122 assert(((
Def->getOperand(DefIdx).isDef() &&
2123 (DefIdx < Def->
getDesc().getNumDefs() ||
2124 Def->getDesc().isVariadic())) ||
2125 Def->getOperand(DefIdx).isImplicit()) &&
2128 return getNextSourceFromCopy();
2129 if (
Def->isBitcast())
2130 return getNextSourceFromBitcast();
2134 return ValueTrackerResult();
2135 if (
Def->isRegSequence() ||
Def->isRegSequenceLike())
2136 return getNextSourceFromRegSequence();
2137 if (
Def->isInsertSubreg() ||
Def->isInsertSubregLike())
2138 return getNextSourceFromInsertSubreg();
2139 if (
Def->isExtractSubreg() ||
Def->isExtractSubregLike())
2140 return getNextSourceFromExtractSubreg();
2141 if (
Def->isSubregToReg())
2142 return getNextSourceFromSubregToReg();
2144 return getNextSourceFromPHI();
2145 return ValueTrackerResult();
2148ValueTrackerResult ValueTracker::getNextSource() {
2152 return ValueTrackerResult();
2154 ValueTrackerResult Res = getNextSourceImpl();
2155 if (Res.isValid()) {
2159 bool OneRegSrc = Res.getNumSources() == 1;
2161 Reg = Res.getSrcReg(0);
2168 if (!
Reg.isPhysical() && OneRegSrc) {
2170 if (DI !=
MRI.def_end()) {
2173 DefSubReg = Res.getSrcSubReg(0);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
A common definition of LaneBitmask for use in TableGen and CodeGen.
unsigned const TargetRegisterInfo * TRI
TargetInstrInfo::RegSubRegPair RegSubRegPair
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
TargetInstrInfo::RegSubRegPair RegSubRegPair
static cl::opt< unsigned > RewritePHILimit("rewrite-phi-limit", cl::Hidden, cl::init(10), cl::desc("Limit the length of PHI chains to lookup"))
static cl::opt< bool > DisablePeephole("disable-peephole", cl::Hidden, cl::init(false), cl::desc("Disable the peephole optimizer"))
static cl::opt< unsigned > MaxRecurrenceChain("recurrence-chain-limit", cl::Hidden, cl::init(3), cl::desc("Maximum length of recurrence chain when evaluating the benefit " "of commuting operands"))
static cl::opt< bool > DisableNAPhysCopyOpt("disable-non-allocatable-phys-copy-opt", cl::Hidden, cl::init(false), cl::desc("Disable non-allocatable physical register copy optimization"))
static bool isVirtualRegisterOperand(MachineOperand &MO)
\bried Returns true if MO is a virtual register operand.
static MachineInstr & insertPHI(MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const SmallVectorImpl< RegSubRegPair > &SrcRegs, MachineInstr &OrigPHI)
Insert a PHI instruction with incoming edges SrcRegs that are guaranteed to have the same register cl...
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
static cl::opt< bool > DisableAdvCopyOpt("disable-adv-copy-opt", cl::Hidden, cl::init(false), cl::desc("Disable advanced copy optimization"))
Specifiy whether or not the value tracking looks through complex instructions.
static RegSubRegPair getNewSource(MachineRegisterInfo *MRI, const TargetInstrInfo *TII, RegSubRegPair Def, const PeepholeOptimizer::RewriteMapTy &RewriteMap, bool HandleMultipleSources=true)
Given a Def.Reg and Def.SubReg pair, use RewriteMap to find the new source to use for rewrite.
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Virtual Register Rewriter
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Represents analyses that only rely on functions' control flow.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &Mask, int64_t &Value) const override
For a comparison instruction, return the source registers in SrcReg and SrcReg2 if having two registe...
bool isLoopHeader(const BlockT *BB) const
Describe properties that are true of each instruction in the target description file.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
An RAII based helper class to modify MachineFunctionProperties when running pass.
Analysis pass which computes a MachineDominatorTree.
Analysis pass which computes a MachineDominatorTree.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
virtual void MF_HandleChangeDesc(MachineInstr &MI, const MCInstrDesc &TID)
Callback before changing MCInstrDesc.
virtual void MF_HandleRemoval(MachineInstr &MI)=0
Callback before a removal. This should not modify the MI directly.
virtual void MF_HandleInsertion(MachineInstr &MI)=0
Callback after an insertion. This should not modify the MI directly.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
void setDelegate(Delegate *delegate)
Set the delegate.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
Analysis pass that exposes the MachineLoopInfo for a machine function.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
MachineBasicBlock * getMBB() const
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
const uint32_t * getRegMask() const
getRegMask - Returns a bit mask of registers preserved by this RegMask operand.
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetInstrInfo - Interface to description of machine instruction set.
static const unsigned CommuteAnyOperandIndex
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
MCInstrDesc const & getDesc(MCInstrInfo const &MCII, MCInst const &MCI)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
std::pair< unsigned, unsigned > IndexPair
The pair of an instruction index and a operand index.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
char & PeepholeOptimizerLegacyID
PeepholeOptimizer - This pass performs peephole optimizations - like extension and comparison elimina...
PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void initializePeepholeOptimizerLegacyPass(PassRegistry &)
Implement std::hash so that hash_code can be used in STL containers.
A pair composed of a pair of a register and a sub-register index, and another sub-register index.
A pair composed of a register and a sub-register index.