104#define DEBUG_TYPE "peephole-opt"
108 cl::desc(
"Aggressive extension optimization"));
110STATISTIC(NumReuse,
"Number of extension results reused");
112STATISTIC(NumImmFold,
"Number of move immediate folded");
115STATISTIC(NumUncoalescableCopies,
"Number of uncoalescable copies optimized");
116STATISTIC(NumRewrittenCopies,
"Number of copies rewritten");
117STATISTIC(NumNAPhysCopies,
"Number of non-allocatable physical copies removed");
121class ValueTrackerResult;
122class RecurrenceInstr;
128 int CurrentSrcIdx = 0;
131 virtual ~Rewriter() =
default;
163 virtual bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg) = 0;
167class CopyRewriter :
public Rewriter {
170 assert(
MI.isCopy() &&
"Expected copy instruction");
172 ~CopyRewriter()
override =
default;
176 if (++CurrentSrcIdx > 1)
180 const MachineOperand &MOSrc = CopyLike.getOperand(CurrentSrcIdx);
183 const MachineOperand &MODef = CopyLike.getOperand(0);
188 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
189 MachineOperand &MOSrc = CopyLike.getOperand(CurrentSrcIdx);
198class UncoalescableRewriter :
public Rewriter {
202 UncoalescableRewriter(MachineInstr &
MI) :
Rewriter(
MI) {
203 NumDefs =
MI.getDesc().getNumDefs();
213 if (CurrentSrcIdx == NumDefs)
216 while (CopyLike.getOperand(CurrentSrcIdx).isDead()) {
218 if (CurrentSrcIdx == NumDefs)
224 const MachineOperand &MODef = CopyLike.getOperand(CurrentSrcIdx);
231 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
237class InsertSubregRewriter :
public Rewriter {
240 assert(
MI.isInsertSubreg() &&
"Invalid instruction");
257 if (CurrentSrcIdx == 2)
261 const MachineOperand &MOInsertedReg = CopyLike.getOperand(2);
263 const MachineOperand &MODef = CopyLike.getOperand(0);
271 (
unsigned)CopyLike.getOperand(3).getImm());
275 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
276 if (CurrentSrcIdx != 2)
279 MachineOperand &MO = CopyLike.getOperand(CurrentSrcIdx);
287class ExtractSubregRewriter :
public Rewriter {
288 const TargetInstrInfo &TII;
291 ExtractSubregRewriter(MachineInstr &
MI,
const TargetInstrInfo &TII)
293 assert(
MI.isExtractSubreg() &&
"Invalid instruction");
304 if (CurrentSrcIdx == 1)
308 const MachineOperand &MOExtractedReg = CopyLike.getOperand(1);
317 const MachineOperand &MODef = CopyLike.getOperand(0);
322 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
324 if (CurrentSrcIdx != 1)
327 CopyLike.getOperand(CurrentSrcIdx).setReg(NewReg);
338 CopyLike.removeOperand(2);
340 CopyLike.setDesc(TII.get(TargetOpcode::COPY));
343 CopyLike.getOperand(CurrentSrcIdx + 1).setImm(NewSubReg);
349class RegSequenceRewriter :
public Rewriter {
352 assert(
MI.isRegSequence() &&
"Invalid instruction");
376 if (
static_cast<unsigned>(CurrentSrcIdx) >= CopyLike.getNumOperands())
379 const MachineOperand &MOInsertedReg = CopyLike.getOperand(CurrentSrcIdx);
380 Src.Reg = MOInsertedReg.
getReg();
385 Dst.SubReg = CopyLike.getOperand(CurrentSrcIdx + 1).getImm();
387 const MachineOperand &MODef = CopyLike.getOperand(0);
389 assert(MODef.
getSubReg() == 0 &&
"cannot have subregister def in SSA");
393 bool RewriteCurrentSource(
Register NewReg,
unsigned NewSubReg)
override {
394 MachineOperand &MO = CopyLike.getOperand(CurrentSrcIdx);
402 const CodeGenOptions &Opts;
403 const TargetInstrInfo *TII =
nullptr;
404 const TargetRegisterInfo *TRI =
nullptr;
405 MachineRegisterInfo *MRI =
nullptr;
406 MachineDominatorTree *DT =
nullptr;
407 MachineLoopInfo *MLI =
nullptr;
410 PeepholeOptimizer(
const CodeGenOptions &Opts, MachineDominatorTree *DT,
411 MachineLoopInfo *MLI)
412 : Opts(Opts), DT(DT), MLI(MLI) {}
416 using RewriteMapTy = SmallDenseMap<RegSubRegPair, ValueTrackerResult>;
419 using RecurrenceCycle = SmallVector<RecurrenceInstr, 4>;
423 SmallPtrSet<MachineInstr *, 16> &LocalMIs);
424 bool optimizeExtInstr(MachineInstr &
MI, MachineBasicBlock &
MBB,
425 SmallPtrSetImpl<MachineInstr *> &LocalMIs);
426 bool optimizeSelect(MachineInstr &
MI,
427 SmallPtrSetImpl<MachineInstr *> &LocalMIs);
428 bool optimizeCondBranch(MachineInstr &
MI);
430 bool optimizeCoalescableCopyImpl(
Rewriter &&CpyRewriter);
431 bool optimizeCoalescableCopy(MachineInstr &
MI);
432 bool optimizeUncoalescableCopy(MachineInstr &
MI,
433 SmallPtrSetImpl<MachineInstr *> &LocalMIs);
434 bool optimizeRecurrence(MachineInstr &
PHI);
437 bool isMoveImmediate(MachineInstr &
MI, SmallSet<Register, 4> &ImmDefRegs,
438 DenseMap<Register, MachineInstr *> &ImmDefMIs);
439 bool foldImmediate(MachineInstr &
MI, SmallSet<Register, 4> &ImmDefRegs,
440 DenseMap<Register, MachineInstr *> &ImmDefMIs,
448 const SmallSet<Register, 2> &TargetReg,
449 RecurrenceCycle &RC);
456 bool foldRedundantCopy(MachineInstr &
MI);
467 foldRedundantNAPhysCopy(MachineInstr &
MI,
468 DenseMap<Register, MachineInstr *> &NAPhysToVirtMIs);
470 bool isLoadFoldable(MachineInstr &
MI,
471 SmallSet<Register, 16> &FoldAsLoadDefCandidates);
478 SmallPtrSet<MachineInstr *, 16> &LocalMIs);
482 bool isCoalescableCopy(
const MachineInstr &
MI)
const {
485 return MI.isCopy() ||
486 (Opts.adv_copy_opt && (
MI.isRegSequence() ||
MI.isInsertSubreg() ||
487 MI.isExtractSubreg()));
492 bool isUncoalescableCopy(
const MachineInstr &
MI)
const {
493 return MI.isBitcast() || (Opts.adv_copy_opt && (
MI.isRegSequenceLike() ||
494 MI.isInsertSubregLike() ||
495 MI.isExtractSubregLike()));
498 MachineInstr &rewriteSource(MachineInstr &CopyLike,
RegSubRegPair Def,
499 RewriteMapTy &RewriteMap);
503 DenseMap<RegSubRegPair, MachineInstr *> CopySrcMIs;
506 void MF_HandleInsertion(MachineInstr &
MI)
override {}
513 unsigned SrcSubReg =
MI.getOperand(1).getSubReg();
514 if (!SrcReg.
isVirtual() && !MRI->isConstantPhysReg(SrcReg))
523 void deleteChangedCopy(MachineInstr &
MI) {
525 if (!getCopySrc(
MI, SrcPair))
528 auto It = CopySrcMIs.find(SrcPair);
529 if (It != CopySrcMIs.end() && It->second == &
MI)
530 CopySrcMIs.erase(It);
533 void MF_HandleRemoval(MachineInstr &
MI)
override { deleteChangedCopy(
MI); }
535 void MF_HandleChangeDesc(MachineInstr &
MI,
const MCInstrDesc &TID)
override {
536 deleteChangedCopy(
MI);
544 PeepholeOptimizerLegacy() : MachineFunctionPass(ID) {}
548 void getAnalysisUsage(AnalysisUsage &AU)
const override {
557 MachineFunctionProperties getRequiredProperties()
const override {
558 return MachineFunctionProperties().setIsSSA();
568class RecurrenceInstr {
570 using IndexPair = std::pair<unsigned, unsigned>;
572 RecurrenceInstr(MachineInstr *MI) : MI(MI) {}
573 RecurrenceInstr(MachineInstr *MI,
unsigned Idx1,
unsigned Idx2)
574 : MI(MI), CommutePair(std::make_pair(Idx1, Idx2)) {}
576 MachineInstr *getMI()
const {
return MI; }
577 std::optional<IndexPair> getCommutePair()
const {
return CommutePair; }
581 std::optional<IndexPair> CommutePair;
587class ValueTrackerResult {
593 const MachineInstr *Inst =
nullptr;
596 ValueTrackerResult() =
default;
598 ValueTrackerResult(
Register Reg,
unsigned SubReg) { addSource(
Reg, SubReg); }
600 bool isValid()
const {
return getNumSources() > 0; }
602 void setInst(
const MachineInstr *
I) { Inst =
I; }
603 const MachineInstr *getInst()
const {
return Inst; }
610 void addSource(
Register SrcReg,
unsigned SrcSubReg) {
614 void setSource(
int Idx,
Register SrcReg,
unsigned SrcSubReg) {
615 assert(Idx < getNumSources() &&
"Reg pair source out of index");
619 int getNumSources()
const {
return RegSrcs.size(); }
624 assert(Idx < getNumSources() &&
"Reg source out of index");
625 return RegSrcs[
Idx].Reg;
628 unsigned getSrcSubReg(
int Idx)
const {
629 assert(Idx < getNumSources() &&
"SubReg source out of index");
630 return RegSrcs[
Idx].SubReg;
634 if (
Other.getInst() != getInst())
637 if (
Other.getNumSources() != getNumSources())
640 for (
int i = 0, e =
Other.getNumSources(); i != e; ++i)
641 if (
Other.getSrcReg(i) != getSrcReg(i) ||
642 Other.getSrcSubReg(i) != getSrcSubReg(i))
666 const CodeGenOptions &Opts;
669 const MachineInstr *Def =
nullptr;
681 const MachineRegisterInfo &MRI;
684 const TargetInstrInfo *TII;
687 ValueTrackerResult getNextSourceImpl();
690 ValueTrackerResult getNextSourceFromCopy();
693 ValueTrackerResult getNextSourceFromBitcast();
696 ValueTrackerResult getNextSourceFromRegSequence();
699 ValueTrackerResult getNextSourceFromInsertSubreg();
702 ValueTrackerResult getNextSourceFromExtractSubreg();
705 ValueTrackerResult getNextSourceFromSubregToReg();
708 ValueTrackerResult getNextSourceFromPHI();
720 ValueTracker(
const CodeGenOptions &Opts,
Register Reg,
unsigned DefSubReg,
721 const MachineRegisterInfo &MRI,
722 const TargetInstrInfo *TII =
nullptr)
723 : Opts(Opts), DefSubReg(DefSubReg), Reg(Reg), MRI(MRI), TII(TII) {
724 if (!Reg.isPhysical()) {
725 MachineRegisterInfo::def_iterator DI = MRI.def_begin(Reg);
726 if (DI != MRI.def_end()) {
727 Def = DI->getParent();
728 DefIdx = DI.getOperandNo();
738 ValueTrackerResult getNextSource();
743char PeepholeOptimizerLegacy::ID = 0;
748 "Peephole Optimizations",
false,
false)
762bool PeepholeOptimizer::optimizeExtInstr(
767 if (!
TII->isCoalescableExtInstr(
MI, SrcReg, DstReg, SubIdx))
773 if (
MRI->hasOneNonDBGUse(SrcReg))
780 DstRC =
TRI->getSubClassWithSubReg(DstRC, SubIdx);
790 TRI->getSubClassWithSubReg(
MRI->getRegClass(SrcReg), SubIdx) !=
nullptr;
796 ReachedBBs.insert(UI.getParent());
804 bool ExtendLife =
true;
806 MachineInstr *UseMI = UseMO.getParent();
810 if (UseMI->isPHI()) {
816 if (UseSrcSubIdx && UseMO.getSubReg() != SubIdx)
836 if (
UseMI->getOpcode() == TargetOpcode::SUBREG_TO_REG)
840 if (UseMBB == &
MBB) {
842 if (!LocalMIs.count(
UseMI))
843 Uses.push_back(&UseMO);
844 }
else if (ReachedBBs.count(UseMBB)) {
847 Uses.push_back(&UseMO);
851 ExtendedUses.push_back(&UseMO);
860 if (ExtendLife && !ExtendedUses.empty())
862 Uses.append(ExtendedUses.begin(), ExtendedUses.end());
867 SmallPtrSet<MachineBasicBlock *, 4> PHIBBs;
872 for (MachineInstr &UI : MRI->use_nodbg_instructions(DstReg))
874 PHIBBs.insert(UI.getParent());
876 const TargetRegisterClass *RC = MRI->getRegClass(SrcReg);
877 for (MachineOperand *UseMO : Uses) {
878 MachineInstr *UseMI = UseMO->getParent();
879 MachineBasicBlock *UseMBB = UseMI->getParent();
880 if (PHIBBs.count(UseMBB))
884 MRI->constrainRegClass(DstReg, DstRC);
901 RC = MRI->getRegClass(UseMO->getReg());
902 if (UseMO->getSubReg())
903 RC = TRI->getSubRegisterClass(RC, UseMO->getSubReg());
906 Register NewVR = MRI->createVirtualRegister(RC);
907 [[maybe_unused]] auto Copy = BuildMI(*UseMBB, UseMI, UseMI->getDebugLoc(),
908 TII->get(TargetOpcode::COPY), NewVR)
909 .addReg(DstReg, {}, SubIdx);
910 LLVM_DEBUG(dbgs() <<
" Build new copy: " << *Copy
911 <<
" Changing: " << *UseMI);
915 UseMO->setReg(NewVR);
916 LLVM_DEBUG(dbgs() <<
" to: " << *UseMI);
929bool PeepholeOptimizer::optimizeCmpInstr(
935 int64_t CmpMask, CmpValue;
942 if (!
TII->optimizeCompareInstr(
MI, SrcReg, SrcReg2, CmpMask, CmpValue,
MRI))
951 if (MachineInstr *FlagProducer =
959 FlagProducer->getIterator()),
960 [](
const MachineInstr &
I) { return I.isLoadFoldBarrier(); }))
961 foldLoadInto(MF, *FlagProducer, SrcReg, LocalMIs);
968bool PeepholeOptimizer::optimizeSelect(
969 MachineInstr &
MI, SmallPtrSetImpl<MachineInstr *> &LocalMIs) {
970 assert(
MI.isSelect() &&
"Should only be called when MI->isSelect() is true");
971 if (!
TII->optimizeSelect(
MI, LocalMIs))
974 MI.eraseFromParent();
980bool PeepholeOptimizer::optimizeCondBranch(MachineInstr &
MI) {
981 return TII->optimizeCondBranch(
MI);
1000 RewriteMapTy &RewriteMap) {
1009 unsigned PHICount = 0;
1013 bool FoundSuitable =
false;
1015 bool Aborted =
false;
1024 ValueTracker ValTracker(Opts, CurSrcPair.
Reg, CurSrcPair.
SubReg, *
MRI,
TII);
1029 ValueTrackerResult Res = ValTracker.getNextSource();
1031 if (!Res.isValid()) {
1037 auto [InsertPt, WasInserted] = RewriteMap.try_emplace(CurSrcPair, Res);
1040 const ValueTrackerResult &CurSrcRes = InsertPt->second;
1042 assert(CurSrcRes == Res &&
"ValueTrackerResult found must match");
1045 if (CurSrcRes.getNumSources() > 1) {
1047 <<
"findNextSource: found PHI cycle, aborting...\n");
1055 unsigned NumSrcs = Res.getNumSources();
1058 if (PHICount >= Opts.rewrite_phi_limit) {
1064 for (
unsigned i = 0; i < NumSrcs; ++i)
1069 CurSrcPair = Res.getSrc(0);
1081 if (!
TRI->shouldRewriteCopySrc(DefRC, DefSubReg, SrcRC,
1087 if (PHICount > 0 && CurSrcPair.
SubReg != 0)
1092 if (CurSrcPair.
SubReg != 0) {
1093 SuitablePair = CurSrcPair;
1094 FoundSuitable =
true;
1105 }
while (!SrcToLook.
empty());
1112 CurSrcPair = SuitablePair;
1113 RewriteMap.erase(SuitablePair);
1117 return CurSrcPair.
Reg !=
Reg;
1129 assert(!SrcRegs.
empty() &&
"No sources to create a PHI instruction?");
1134 assert(SrcRegs[0].SubReg == 0 &&
"should not have subreg operand");
1135 Register NewVR =
MRI.createVirtualRegister(NewRC);
1138 TII.get(TargetOpcode::PHI), NewVR);
1140 unsigned MBBOpIdx = 2;
1142 MIB.
addReg(RegPair.Reg, {}, RegPair.SubReg);
1159 const PeepholeOptimizer::RewriteMapTy &RewriteMap,
1160 bool HandleMultipleSources =
true) {
1163 ValueTrackerResult Res = RewriteMap.
lookup(LookupSrc);
1169 unsigned NumSrcs = Res.getNumSources();
1171 LookupSrc.
Reg = Res.getSrcReg(0);
1172 LookupSrc.
SubReg = Res.getSrcSubReg(0);
1177 if (!HandleMultipleSources)
1183 for (
unsigned i = 0; i < NumSrcs; ++i) {
1184 RegSubRegPair PHISrc(Res.getSrcReg(i), Res.getSrcSubReg(i));
1202bool PeepholeOptimizer::optimizeCoalescableCopyImpl(
Rewriter &&CpyRewriter) {
1208 while (CpyRewriter.getNextRewritableSource(TrackPair, Dst)) {
1209 if (Dst.Reg.isPhysical()) {
1220 RewriteMapTy RewriteMap;
1223 if (!findNextSource(DefRC, Dst.SubReg, TrackPair, RewriteMap))
1231 "should not rewrite source to original value");
1241 TRI->getSubClassWithSubReg(RC, NewSrc.
SubReg);
1248 if (CpyRewriter.RewriteCurrentSource(NewSrc.
Reg, NewSrc.
SubReg))
1257 NumRewrittenCopies +=
Changed;
1272bool PeepholeOptimizer::optimizeCoalescableCopy(MachineInstr &
MI) {
1273 assert(isCoalescableCopy(
MI) &&
"Invalid argument");
1274 assert(
MI.getDesc().getNumDefs() == 1 &&
1275 "Coalescer can understand multiple defs?!");
1276 const MachineOperand &MODef =
MI.getOperand(0);
1281 switch (
MI.getOpcode()) {
1282 case TargetOpcode::COPY:
1283 return optimizeCoalescableCopyImpl(CopyRewriter(
MI));
1284 case TargetOpcode::INSERT_SUBREG:
1285 return optimizeCoalescableCopyImpl(InsertSubregRewriter(
MI));
1286 case TargetOpcode::EXTRACT_SUBREG:
1287 return optimizeCoalescableCopyImpl(ExtractSubregRewriter(
MI, *
TII));
1288 case TargetOpcode::REG_SEQUENCE:
1289 return optimizeCoalescableCopyImpl(RegSequenceRewriter(
MI));
1292 if (
MI.isBitcast() ||
MI.isRegSequenceLike() ||
MI.isInsertSubregLike() ||
1293 MI.isExtractSubregLike())
1294 return optimizeCoalescableCopyImpl(UncoalescableRewriter(
MI));
1304MachineInstr &PeepholeOptimizer::rewriteSource(MachineInstr &CopyLike,
1306 RewriteMapTy &RewriteMap) {
1307 assert(!
Def.Reg.isPhysical() &&
"We do not rewrite physical registers");
1319 TRI->getSubClassWithSubReg(NewSrcRC, NewSrc.
SubReg);
1328 MachineInstr *NewCopy =
1330 TII->get(TargetOpcode::COPY), NewVReg)
1357bool PeepholeOptimizer::optimizeUncoalescableCopy(
1358 MachineInstr &
MI, SmallPtrSetImpl<MachineInstr *> &LocalMIs) {
1359 assert(isUncoalescableCopy(
MI) &&
"Invalid argument");
1360 UncoalescableRewriter CpyRewriter(
MI);
1365 RewriteMapTy RewriteMap;
1369 while (CpyRewriter.getNextRewritableSource(Src, Def)) {
1372 if (
Def.Reg.isPhysical())
1382 if (!findNextSource(DefRC,
Def.SubReg, Def, RewriteMap))
1391 MachineInstr &NewCopy = rewriteSource(
MI, Def, RewriteMap);
1392 LocalMIs.
insert(&NewCopy);
1397 MI.eraseFromParent();
1398 ++NumUncoalescableCopies;
1405bool PeepholeOptimizer::isLoadFoldable(
1406 MachineInstr &
MI, SmallSet<Register, 16> &FoldAsLoadDefCandidates) {
1407 if (!
MI.canFoldAsLoad() || !
MI.mayLoad())
1409 const MCInstrDesc &MCID =
MI.getDesc();
1428 SmallPtrSet<MachineInstr *, 16> &LocalMIs) {
1430 MachineInstr *
DefMI =
nullptr;
1431 MachineInstr *CopyMI =
nullptr;
1432 MachineInstr *FoldMI =
TII->optimizeLoadInstr(
MI,
MRI,
Reg,
DefMI, CopyMI);
1441 if (
MI.shouldUpdateAdditionalCallInfo())
1443 MI.eraseFromParent();
1450bool PeepholeOptimizer::isMoveImmediate(
1451 MachineInstr &
MI, SmallSet<Register, 4> &ImmDefRegs,
1452 DenseMap<Register, MachineInstr *> &ImmDefMIs) {
1453 const MCInstrDesc &MCID =
MI.getDesc();
1454 if (MCID.
getNumDefs() != 1 || !
MI.getOperand(0).isReg())
1461 if (!
MI.isMoveImmediate() && !
TII->getConstValDefinedInReg(
MI,
Reg, ImmVal))
1472bool PeepholeOptimizer::foldImmediate(
1473 MachineInstr &
MI, SmallSet<Register, 4> &ImmDefRegs,
1474 DenseMap<Register, MachineInstr *> &ImmDefMIs,
bool &
Deleted) {
1476 for (
unsigned i = 0, e =
MI.getDesc().getNumOperands(); i != e; ++i) {
1477 MachineOperand &MO =
MI.getOperand(i);
1486 assert(
II != ImmDefMIs.
end() &&
"couldn't find immediate definition");
1498 MI.eraseFromParent();
1522bool PeepholeOptimizer::foldRedundantCopy(MachineInstr &
MI) {
1523 assert(
MI.isCopy() &&
"expected a COPY machine instruction");
1526 if (!getCopySrc(
MI, SrcPair))
1533 if (CopySrcMIs.
insert(std::make_pair(SrcPair, &
MI)).second) {
1538 MachineInstr *PrevCopy = CopySrcMIs.
find(SrcPair)->second;
1541 "Unexpected mismatching subreg!");
1556bool PeepholeOptimizer::isNAPhysCopy(
Register Reg) {
1560bool PeepholeOptimizer::foldRedundantNAPhysCopy(
1561 MachineInstr &
MI, DenseMap<Register, MachineInstr *> &NAPhysToVirtMIs) {
1562 assert(
MI.isCopy() &&
"expected a COPY machine instruction");
1564 if (!Opts.non_allocatable_phys_copy_opt)
1569 if (isNAPhysCopy(SrcReg) && DstReg.
isVirtual()) {
1573 NAPhysToVirtMIs.
insert({SrcReg, &
MI});
1577 if (!(SrcReg.
isVirtual() && isNAPhysCopy(DstReg)))
1581 auto PrevCopy = NAPhysToVirtMIs.
find(DstReg);
1582 if (PrevCopy == NAPhysToVirtMIs.
end()) {
1585 LLVM_DEBUG(
dbgs() <<
"NAPhysCopy: intervening clobber forbids erasing "
1591 if (PrevDstReg == SrcReg) {
1604 NAPhysToVirtMIs.
erase(PrevCopy);
1613bool PeepholeOptimizer::findTargetRecurrence(
1614 Register Reg,
const SmallSet<Register, 2> &TargetRegs,
1615 RecurrenceCycle &RC) {
1629 if (RC.size() >= Opts.recurrence_chain_limit)
1633 unsigned Idx =
MI.findRegisterUseOperandIdx(
Reg,
nullptr);
1637 if (
MI.getDesc().getNumDefs() != 1)
1640 MachineOperand &DefOp =
MI.getOperand(0);
1647 unsigned TiedUseIdx;
1648 if (!
MI.isRegTiedToUseOperand(0, &TiedUseIdx))
1651 if (Idx == TiedUseIdx) {
1652 RC.push_back(RecurrenceInstr(&
MI));
1653 return findTargetRecurrence(DefOp.
getReg(), TargetRegs, RC);
1657 if (
TII->findCommutedOpIndices(
MI, Idx, CommIdx) && CommIdx == TiedUseIdx) {
1658 RC.push_back(RecurrenceInstr(&
MI, Idx, CommIdx));
1659 return findTargetRecurrence(DefOp.
getReg(), TargetRegs, RC);
1684bool PeepholeOptimizer::optimizeRecurrence(MachineInstr &
PHI) {
1685 SmallSet<Register, 2> TargetRegs;
1686 for (
unsigned Idx = 1;
Idx <
PHI.getNumOperands();
Idx += 2) {
1687 MachineOperand &MO =
PHI.getOperand(Idx);
1694 if (findTargetRecurrence(
PHI.getOperand(0).getReg(), TargetRegs, RC)) {
1698 for (
auto &RI : RC) {
1700 auto CP = RI.getCommutePair();
1703 TII->commuteInstruction(*(RI.getMI()),
false, (*CP).first,
1730bool PeepholeOptimizerLegacy::runOnMachineFunction(
MachineFunction &MF) {
1734 ? &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree()
1736 auto *MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
1738 return Impl.run(MF);
1743 LLVM_DEBUG(
dbgs() <<
"********** PEEPHOLE OPTIMIZER **********\n");
1757 bool SeenMoveImm =
false;
1790 if (
MI->isDebugInstr())
1793 if (
MI->isPosition())
1796 if (IsLoopHeader &&
MI->isPHI()) {
1797 if (optimizeRecurrence(*
MI)) {
1803 if (!
MI->isCopy()) {
1804 for (
const MachineOperand &MO :
MI->operands()) {
1808 if (MO.
isDef() && isNAPhysCopy(
Reg)) {
1810 if (Def != NAPhysToVirtMIs.
end()) {
1814 <<
"NAPhysCopy: invalidating because of " << *
MI);
1815 NAPhysToVirtMIs.
erase(Def);
1820 NAPhysToVirtMIs.
remove_if([&](
const auto &RegMI) {
1824 <<
"NAPhysCopy: invalidating because of " << *
MI);
1831 if (
MI->isImplicitDef() ||
MI->isKill())
1834 if (
MI->isInlineAsm() ||
MI->hasUnmodeledSideEffects()) {
1841 NAPhysToVirtMIs.
clear();
1844 if (
MI->isCompare() && optimizeCmpInstr(*
MI, MF, LocalMIs)) {
1849 if ((isUncoalescableCopy(*
MI) &&
1850 optimizeUncoalescableCopy(*
MI, LocalMIs)) ||
1851 (
MI->isSelect() && optimizeSelect(*
MI, LocalMIs))) {
1858 if (
MI->isConditionalBranch() && optimizeCondBranch(*
MI)) {
1863 if (isCoalescableCopy(*
MI) && optimizeCoalescableCopy(*
MI)) {
1869 if (
MI->isCopy() && (foldRedundantCopy(*
MI) ||
1870 foldRedundantNAPhysCopy(*
MI, NAPhysToVirtMIs))) {
1873 MI->eraseFromParent();
1878 if (isMoveImmediate(*
MI, ImmDefRegs, ImmDefMIs)) {
1900 if (!isLoadFoldable(*
MI, FoldAsLoadDefCandidates) &&
1901 !FoldAsLoadDefCandidates.
empty()) {
1908 const MCInstrDesc &MIDesc =
MI->getDesc();
1909 for (
unsigned i = MIDesc.
getNumDefs(); i !=
MI->getNumOperands(); ++i) {
1910 const MachineOperand &MOp =
MI->getOperand(i);
1914 if (FoldAsLoadDefCandidates.
count(FoldAsLoadDefReg)) {
1917 Register FoldedReg = FoldAsLoadDefReg;
1918 if (MachineInstr *FoldMI =
1919 foldLoadInto(MF, *
MI, FoldAsLoadDefReg, LocalMIs)) {
1920 FoldAsLoadDefCandidates.
erase(FoldedReg);
1932 if (
MI->isLoadFoldBarrier()) {
1934 FoldAsLoadDefCandidates.
clear();
1939 MF.resetDelegate(
this);
1943ValueTrackerResult ValueTracker::getNextSourceFromCopy() {
1944 assert(
Def->isCopy() &&
"Invalid definition");
1949 assert(
Def->getNumOperands() -
Def->getNumImplicitOperands() == 2 &&
1950 "Invalid number of operands");
1951 assert(!
Def->hasImplicitDef() &&
"Only implicit uses are allowed");
1952 assert(!
Def->getOperand(DefIdx).getSubReg() &&
"no subregister defs in SSA");
1955 const MachineOperand &Src =
Def->getOperand(1);
1957 return ValueTrackerResult();
1960 unsigned SubReg = Src.getSubReg();
1963 SubReg =
TRI->composeSubRegIndices(SubReg, DefSubReg);
1968 if (!
TRI->isSubRegValidForRegClass(RegRC, SubReg))
1969 return ValueTrackerResult();
1971 if (!
TRI->getSubReg(SrcReg, SubReg))
1972 return ValueTrackerResult();
1976 return ValueTrackerResult(SrcReg, SubReg);
1979ValueTrackerResult ValueTracker::getNextSourceFromBitcast() {
1980 assert(
Def->isBitcast() &&
"Invalid definition");
1983 if (
Def->mayRaiseFPException() ||
Def->hasUnmodeledSideEffects())
1984 return ValueTrackerResult();
1987 if (
Def->getDesc().getNumDefs() != 1)
1988 return ValueTrackerResult();
1990 assert(!
Def->getOperand(DefIdx).getSubReg() &&
"no subregister defs in SSA");
1992 unsigned SrcIdx =
Def->getNumOperands();
1993 for (
unsigned OpIdx = DefIdx + 1, EndOpIdx = SrcIdx; OpIdx != EndOpIdx;
1995 const MachineOperand &MO =
Def->getOperand(OpIdx);
2001 assert(!MO.
isDef() &&
"We should have skipped all the definitions by now");
2002 if (SrcIdx != EndOpIdx)
2004 return ValueTrackerResult();
2010 if (SrcIdx >=
Def->getNumOperands())
2011 return ValueTrackerResult();
2013 const MachineOperand &DefOp =
Def->getOperand(DefIdx);
2018 if (
UseMI.isSubregToReg())
2019 return ValueTrackerResult();
2022 const MachineOperand &Src =
Def->getOperand(SrcIdx);
2024 return ValueTrackerResult();
2025 return ValueTrackerResult(Src.getReg(), Src.getSubReg());
2028ValueTrackerResult ValueTracker::getNextSourceFromRegSequence() {
2029 assert((
Def->isRegSequence() ||
Def->isRegSequenceLike()) &&
2030 "Invalid definition");
2032 assert(!
Def->getOperand(DefIdx).getSubReg() &&
"illegal subregister def");
2035 if (!
TII->getRegSequenceInputs(*Def, DefIdx, RegSeqInputRegs))
2036 return ValueTrackerResult();
2044 if (RegSeqInput.SubIdx == DefSubReg)
2045 return ValueTrackerResult(RegSeqInput.Reg, RegSeqInput.SubReg);
2053 LaneBitmask DefMask =
TRI->getSubRegIndexLaneMask(DefSubReg);
2054 LaneBitmask ThisOpRegMask =
TRI->getSubRegIndexLaneMask(RegSeqInput.SubIdx);
2060 if ((DefMask & ThisOpRegMask) != DefMask)
2063 unsigned ReverseDefCompose =
2064 TRI->reverseComposeSubRegIndices(RegSeqInput.SubIdx, DefSubReg);
2065 if (!ReverseDefCompose)
2068 unsigned ComposedDefInSrcReg1 =
2069 TRI->composeSubRegIndices(RegSeqInput.SubReg, ReverseDefCompose);
2076 if (!
TRI->isSubRegValidForRegClass(SrcRC, ComposedDefInSrcReg1))
2077 return ValueTrackerResult();
2079 return ValueTrackerResult(RegSeqInput.Reg, ComposedDefInSrcReg1);
2085 return ValueTrackerResult();
2088ValueTrackerResult ValueTracker::getNextSourceFromInsertSubreg() {
2089 assert((
Def->isInsertSubreg() ||
Def->isInsertSubregLike()) &&
2090 "Invalid definition");
2091 assert(!
Def->getOperand(DefIdx).getSubReg() &&
"no subreg defs in SSA");
2095 if (!
TII->getInsertSubregInputs(*Def, DefIdx, BaseReg, InsertedReg))
2096 return ValueTrackerResult();
2105 if (InsertedReg.
SubIdx == DefSubReg) {
2106 return ValueTrackerResult(InsertedReg.
Reg, InsertedReg.
SubReg);
2111 const MachineOperand &MODef =
Def->getOperand(DefIdx);
2117 return ValueTrackerResult();
2122 if ((
TRI->getSubRegIndexLaneMask(DefSubReg) &
2123 TRI->getSubRegIndexLaneMask(InsertedReg.
SubIdx))
2125 return ValueTrackerResult();
2128 return ValueTrackerResult(
BaseReg.Reg, DefSubReg);
2131ValueTrackerResult ValueTracker::getNextSourceFromExtractSubreg() {
2132 assert((
Def->isExtractSubreg() ||
Def->isExtractSubregLike()) &&
2133 "Invalid definition");
2140 return ValueTrackerResult();
2143 if (!
TII->getExtractSubregInputs(*Def, DefIdx, ExtractSubregInputReg))
2144 return ValueTrackerResult();
2148 if (ExtractSubregInputReg.
SubReg)
2149 return ValueTrackerResult();
2151 return ValueTrackerResult(ExtractSubregInputReg.
Reg,
2152 ExtractSubregInputReg.
SubIdx);
2155ValueTrackerResult ValueTracker::getNextSourceFromSubregToReg() {
2156 assert(
Def->isSubregToReg() &&
"Invalid definition");
2164 if (DefSubReg !=
Def->getOperand(2).getImm())
2165 return ValueTrackerResult();
2168 if (
Def->getOperand(1).getSubReg())
2169 return ValueTrackerResult();
2171 return ValueTrackerResult(
Def->getOperand(1).getReg(),
2172 Def->getOperand(2).getImm());
2176ValueTrackerResult ValueTracker::getNextSourceFromPHI() {
2177 assert(
Def->isPHI() &&
"Invalid definition");
2178 ValueTrackerResult Res;
2181 for (
unsigned i = 1, e =
Def->getNumOperands(); i < e; i += 2) {
2182 const MachineOperand &MO =
Def->getOperand(i);
2187 return ValueTrackerResult();
2194ValueTrackerResult ValueTracker::getNextSourceImpl() {
2195 assert(Def &&
"This method needs a valid definition");
2197 assert(((
Def->getOperand(DefIdx).isDef() &&
2198 (DefIdx < Def->
getDesc().getNumDefs() ||
2199 Def->getDesc().isVariadic())) ||
2200 Def->getOperand(DefIdx).isImplicit()) &&
2203 return getNextSourceFromCopy();
2204 if (
Def->isBitcast())
2205 return getNextSourceFromBitcast();
2208 if (!Opts.adv_copy_opt)
2209 return ValueTrackerResult();
2210 if (
Def->isRegSequence() ||
Def->isRegSequenceLike())
2211 return getNextSourceFromRegSequence();
2212 if (
Def->isInsertSubreg() ||
Def->isInsertSubregLike())
2213 return getNextSourceFromInsertSubreg();
2214 if (
Def->isExtractSubreg() ||
Def->isExtractSubregLike())
2215 return getNextSourceFromExtractSubreg();
2216 if (
Def->isSubregToReg())
2217 return getNextSourceFromSubregToReg();
2219 return getNextSourceFromPHI();
2220 return ValueTrackerResult();
2223ValueTrackerResult ValueTracker::getNextSource() {
2227 return ValueTrackerResult();
2229 ValueTrackerResult Res = getNextSourceImpl();
2230 if (Res.isValid()) {
2234 bool OneRegSrc = Res.getNumSources() == 1;
2236 Reg = Res.getSrcReg(0);
2248 DefSubReg = Res.getSrcSubReg(0);
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file defines the DenseMap class.
const HexagonInstrInfo * TII
A common definition of LaneBitmask for use in TableGen and CodeGen.
TargetInstrInfo::RegSubRegPair RegSubRegPair
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static bool isVirtualRegisterOperand(MachineOperand &MO)
\bried Returns true if MO is a virtual register operand.
static MachineInstr & insertPHI(MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const SmallVectorImpl< RegSubRegPair > &SrcRegs, MachineInstr &OrigPHI)
Insert a PHI instruction with incoming edges SrcRegs that are guaranteed to have the same register cl...
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
TargetInstrInfo::RegSubRegPairAndIdx RegSubRegPairAndIdx
static RegSubRegPair getNewSource(MachineRegisterInfo *MRI, const TargetInstrInfo *TII, RegSubRegPair Def, const PeepholeOptimizer::RewriteMapTy &RewriteMap, bool HandleMultipleSources=true)
Given a Def.Reg and Def.SubReg pair, use RewriteMap to find the new source to use for rewrite.
MachineRegisterInfo & MRI
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
virt reg Virtual Register Rewriter
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Represents analyses that only rely on functions' control flow.
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
bool remove_if(Predicate Pred)
Remove entries that match the given predicate.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &Mask, int64_t &Value) const override
For a comparison instruction, return the source registers in SrcReg and SrcReg2 if having two registe...
bool isLoopHeader(const BlockT *BB) const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
An RAII based helper class to modify MachineFunctionProperties when running pass.
MachineInstrBundleIterator< MachineInstr > iterator
Analysis pass which computes a MachineDominatorTree.
Analysis pass which computes a MachineDominatorTree.
bool dominates(const MachineInstr *A, const MachineInstr *B) const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
void moveAdditionalCallInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
void setDelegate(Delegate *delegate)
Set the delegate.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
const MachineBasicBlock * getParent() const
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
LLVM_ABI bool isIdenticalTo(const MachineInstr &Other, MICheckType Check=CheckDefs) const
Return true if this instruction is identical to Other.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
bool canFoldAsLoad(QueryType Type=IgnoreBundle) const
Return true for instructions that can be folded as memory operands in other instructions.
Analysis pass that exposes the MachineLoopInfo for a machine function.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
MachineBasicBlock * getMBB() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
const uint32_t * getRegMask() const
getRegMask - Returns a bit mask of registers preserved by this RegMask operand.
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
LLVM_ABI void markUsesInDebugValueAsUndef(Register Reg) const
markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the specified register as undefined wh...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
def_iterator def_begin(Register RegNo) const
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
use_instr_nodbg_iterator use_instr_nodbg_begin(Register RegNo) const
LLVM_ABI bool hasOneNonDBGUser(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug instruction using the specified regis...
bool isAllocatable(MCRegister PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
defusechain_iterator< false, true, false, true, false > def_iterator
def_iterator/def_begin/def_end - Walk all defs of the specified register.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
static def_iterator def_end()
const TargetRegisterInfo * getTargetRegisterInfo() const
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
LLVM_ABI MachineInstr * getOneNonDBGUser(Register RegNo) const
If the register has a single non-Debug instruction using the specified register, returns it; otherwis...
LLVM_ABI PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetInstrInfo - Interface to description of machine instruction set.
static const unsigned CommuteAnyOperandIndex
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
MCInstrDesc const & getDesc(MCInstrInfo const &MCII, MCInst const &MCI)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
NodeAddr< DefNode * > Def
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
LLVM_ABI char & PeepholeOptimizerLegacyID
PeepholeOptimizer - This pass performs peephole optimizations - like extension and comparison elimina...
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
const CodeGenOptions & getCodeGenOptions(const LLVMContext &C)
MCRegisterClass TargetRegisterClass
A pair composed of a pair of a register and a sub-register index, and another sub-register index.
A pair composed of a register and a sub-register index.