65#define DEBUG_TYPE "machine-sink"
69 cl::desc(
"Split critical edges during machine sinking"),
74 cl::desc(
"Use block frequency info to find successors to sink"),
78 "machine-sink-split-probability-threshold",
80 "Percentage threshold for splitting single-instruction critical edge. "
81 "If the branch threshold is higher than this threshold, we allow "
82 "speculative execution of up to 1 instruction to avoid branching to "
83 "splitted critical edge"),
87 "machine-sink-load-instrs-threshold",
88 cl::desc(
"Do not try to find alias store for a load if there is a in-path "
89 "block whose instruction number is higher than this threshold."),
93 "machine-sink-load-blocks-threshold",
94 cl::desc(
"Do not try to find alias store for a load if the block number in "
95 "the straight line is higher than this threshold."),
100 cl::desc(
"Sink instructions into cycles to avoid "
105 "machine-sink-cycle-limit",
106 cl::desc(
"The maximum number of instructions considered for cycle sinking."),
109STATISTIC(NumSunk,
"Number of machine instructions sunk");
110STATISTIC(NumCycleSunk,
"Number of machine instructions sunk into a cycle");
113STATISTIC(NumPostRACopySink,
"Number of copies sunk after RA");
139 using AllSuccsCache =
140 std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>;
161 std::map<std::pair<MachineBasicBlock *, MachineBasicBlock *>,
bool>
163 std::map<std::pair<MachineBasicBlock *, MachineBasicBlock *>,
164 std::vector<MachineInstr *>>
168 std::map<MachineBasicBlock *, std::vector<unsigned>> CachedRegisterPressure;
193 CEBCandidates.
clear();
223 AllSuccsCache &AllSuccessors);
233 bool &LocalUse)
const;
235 bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
244 AllSuccsCache &AllSuccessors);
251 AllSuccsCache &AllSuccessors)
const;
258char MachineSinking::ID = 0;
263 "Machine code sinking",
false,
false)
271bool MachineSinking::PerformTrivialForwardCoalescing(
MachineInstr &
MI,
279 !
MRI->hasOneNonDBGUse(SrcReg))
292 MRI->replaceRegWith(DstReg, SrcReg);
293 MI.eraseFromParent();
297 MRI->clearKillFlags(SrcReg);
307bool MachineSinking::AllUsesDominatedByBlock(
Register Reg,
311 bool &LocalUse)
const {
312 assert(
Reg.isVirtual() &&
"Only makes sense for vregs");
315 if (
MRI->use_nodbg_empty(
Reg))
333 MachineInstr *UseInst = MO.getParent();
334 unsigned OpNo = MO.getOperandNo();
335 MachineBasicBlock *UseBlock = UseInst->getParent();
336 return UseBlock == MBB && UseInst->isPHI() &&
337 UseInst->getOperand(OpNo + 1).getMBB() == DefMBB;
346 unsigned OpNo = &MO - &UseInst->
getOperand(0);
348 if (UseInst->
isPHI()) {
352 }
else if (UseBlock == DefMBB) {
368 assert(
MI.mayLoad() &&
"Expected MI that loads!");
372 if (
MI.memoperands_empty())
377 if (PSV->isGOT() || PSV->isConstantPool())
383void MachineSinking::FindCycleSinkCandidates(
386 for (
auto &
MI : *BB) {
389 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction not a candidate for this "
394 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction is not cycle invariant\n");
397 bool DontMoveAcrossStore =
true;
398 if (!
MI.isSafeToMove(AA, DontMoveAcrossStore)) {
399 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction not safe to move.\n");
403 LLVM_DEBUG(
dbgs() <<
"CycleSink: Dont sink GOT or constant pool loads\n");
406 if (
MI.isConvergent())
415 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction added as candidate.\n");
429 DT = &getAnalysis<MachineDominatorTree>();
430 PDT = &getAnalysis<MachinePostDominatorTree>();
431 CI = &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
432 MBFI =
UseBlockFreqInfo ? &getAnalysis<MachineBlockFrequencyInfo>() : nullptr;
433 MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
434 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
437 bool EverMadeChange =
false;
440 bool MadeChange =
false;
443 CEBCandidates.
clear();
449 for (
const auto &Pair : ToSplit) {
450 auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, *
this);
451 if (NewSucc !=
nullptr) {
465 if (!MadeChange)
break;
466 EverMadeChange =
true;
472 for (
auto *
Cycle : Cycles) {
479 FindCycleSinkCandidates(
Cycle, Preheader, Candidates);
487 LLVM_DEBUG(
dbgs() <<
"CycleSink: Limit reached of instructions to "
492 if (!SinkIntoCycle(
Cycle, *
I))
494 EverMadeChange =
true;
500 HasStoreCache.clear();
501 StoreInstrCache.clear();
504 for (
auto I : RegsToClearKillFlags)
505 MRI->clearKillFlags(
I);
506 RegsToClearKillFlags.clear();
508 return EverMadeChange;
520 bool MadeChange =
false;
523 AllSuccsCache AllSuccessors;
528 bool ProcessedBegin, SawStore =
false;
538 if (
MI.isDebugOrPseudoInstr()) {
539 if (
MI.isDebugValue())
544 bool Joined = PerformTrivialForwardCoalescing(
MI, &
MBB);
556 }
while (!ProcessedBegin);
558 SeenDbgUsers.
clear();
561 CachedRegisterPressure.clear();
569 assert(
MI.isDebugValue() &&
"Expected DBG_VALUE for processing");
572 MI.getDebugLoc()->getInlinedAt());
573 bool SeenBefore = SeenDbgVars.
contains(Var);
577 SeenDbgUsers[MO.
getReg()].push_back(SeenDbgUser(&
MI, SeenBefore));
592 if (!CEBCandidates.
insert(std::make_pair(
From, To)).second)
614 if (
Reg.isPhysical())
620 if (
MRI->hasOneNonDBGUse(
Reg)) {
638 if (!isWorthBreakingCriticalEdge(
MI, FromBB, ToBB))
649 if (FromCycle == ToCycle && FromCycle &&
694 if (Pred != FromBB && !DT->
dominates(ToBB, Pred))
698 ToSplit.insert(std::make_pair(FromBB, ToBB));
703std::vector<unsigned> &
709 auto RP = CachedRegisterPressure.find(&
MBB);
710 if (RP != CachedRegisterPressure.end())
724 if (
MI.isDebugInstr() ||
MI.isPseudoProbe())
728 RPTracker.recedeSkipDebugValues();
729 assert(&*RPTracker.getPos() == &
MI &&
"RPTracker sync error!");
730 RPTracker.recede(RegOpers);
733 RPTracker.closeRegion();
734 auto It = CachedRegisterPressure.insert(
735 std::make_pair(&
MBB, RPTracker.getPressure().MaxSetPressure));
736 return It.first->second;
743 AllSuccsCache &AllSuccessors) {
744 assert (SuccToSinkTo &&
"Invalid SinkTo Candidate BB");
746 if (
MBB == SuccToSinkTo)
759 bool NonPHIUse =
false;
762 if (UseBlock == SuccToSinkTo && !UseInst.
isPHI())
770 bool BreakPHIEdge =
false;
773 FindSuccToSinkTo(
MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
774 return isProfitableToSinkTo(
Reg,
MI, SuccToSinkTo, MBB2, AllSuccessors);
784 unsigned Weight =
TRI->getRegClassWeight(RC).RegWeight;
785 const int *PS =
TRI->getRegClassPressureSets(RC);
787 std::vector<unsigned> BBRegisterPressure =
788 getBBRegisterPressure(*SuccToSinkTo);
789 for (; *PS != -1; PS++)
792 if (Weight + BBRegisterPressure[*PS] >=
808 if (
Reg.isPhysical()) {
810 (
MRI->isConstantPhysReg(
Reg) ||
TII->isIgnorableUse(MO)))
820 bool LocalUse =
false;
821 if (!AllUsesDominatedByBlock(
Reg, SuccToSinkTo,
MBB, BreakPHIEdge,
839 if (isRegisterPressureSetExceedLimit(
MRI->getRegClass(
Reg))) {
840 LLVM_DEBUG(
dbgs() <<
"register pressure exceed limit, not profitable.");
856 AllSuccsCache &AllSuccessors)
const {
858 auto Succs = AllSuccessors.find(
MBB);
859 if (Succs != AllSuccessors.end())
860 return Succs->second;
873 if (DTChild->getIDom()->getBlock() ==
MI.getParent() &&
876 AllSuccs.push_back(DTChild->getBlock());
884 bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
885 return HasBlockFreq ? LHSFreq < RHSFreq
889 auto it = AllSuccessors.insert(std::make_pair(
MBB, AllSuccs));
891 return it.first->second;
898 AllSuccsCache &AllSuccessors) {
899 assert (
MBB &&
"Invalid MachineBasicBlock!");
908 if (!MO.
isReg())
continue;
911 if (
Reg == 0)
continue;
913 if (
Reg.isPhysical()) {
918 if (!
MRI->isConstantPhysReg(
Reg) && !
TII->isIgnorableUse(MO))
920 }
else if (!MO.
isDead()) {
926 if (MO.
isUse())
continue;
929 if (!
TII->isSafeToMoveRegClassDefs(
MRI->getRegClass(
Reg)))
937 bool LocalUse =
false;
938 if (!AllUsesDominatedByBlock(
Reg, SuccToSinkTo,
MBB,
939 BreakPHIEdge, LocalUse))
950 GetAllSortedSuccessors(
MI,
MBB, AllSuccessors)) {
951 bool LocalUse =
false;
952 if (AllUsesDominatedByBlock(
Reg, SuccBlock,
MBB,
953 BreakPHIEdge, LocalUse)) {
954 SuccToSinkTo = SuccBlock;
965 if (!isProfitableToSinkTo(
Reg,
MI,
MBB, SuccToSinkTo, AllSuccessors))
972 if (
MBB == SuccToSinkTo)
977 if (SuccToSinkTo && SuccToSinkTo->
isEHPad())
1002 auto *
MBB =
MI.getParent();
1007 auto *PredBB = PredMBB->getBasicBlock();
1013 !PredBB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit))
1018 bool OffsetIsScalable;
1019 if (!
TII->getMemOperandWithOffset(
MI, BaseOp,
Offset, OffsetIsScalable,
TRI))
1022 if (!BaseOp->
isReg())
1025 if (!(
MI.mayLoad() && !
MI.isPredicable()))
1028 MachineBranchPredicate MBP;
1029 if (
TII->analyzeBranchPredicate(*PredMBB, MBP,
false))
1032 return MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
1033 (MBP.Predicate == MachineBranchPredicate::PRED_NE ||
1034 MBP.Predicate == MachineBranchPredicate::PRED_EQ) &&
1035 MBP.LHS.getReg() == BaseOp->
getReg();
1052 auto CopyOperands =
TII.isCopyInstr(SinkInst);
1055 SrcMO = CopyOperands->Source;
1056 DstMO = CopyOperands->Destination;
1059 bool PostRA =
MRI.getNumVirtRegs() == 0;
1067 bool arePhysRegs = !
Reg.isVirtual();
1068 if (arePhysRegs != PostRA)
1075 if (DbgMO.getSubReg() != SrcMO->
getSubReg() ||
1076 DbgMO.getSubReg() != DstMO->getSubReg())
1082 if (PostRA &&
Reg != DstMO->getReg())
1086 DbgMO.setReg(SrcMO->
getReg());
1092using MIRegs = std::pair<MachineInstr *, SmallVector<unsigned, 2>>;
1100 if (!SuccToSinkTo.
empty() && InsertPos != SuccToSinkTo.
end())
1102 InsertPos->getDebugLoc()));
1108 SuccToSinkTo.
splice(InsertPos, ParentBlock,
MI,
1115 for (
const auto &DbgValueToSink : DbgValuesToSink) {
1118 SuccToSinkTo.
insert(InsertPos, NewDbgMI);
1120 bool PropagatedAllSunkOps =
true;
1121 for (
unsigned Reg : DbgValueToSink.second) {
1124 PropagatedAllSunkOps =
false;
1129 if (!PropagatedAllSunkOps)
1143 auto BlockPair = std::make_pair(
From, To);
1147 if (HasStoreCache.find(BlockPair) != HasStoreCache.end())
1148 return HasStoreCache[BlockPair];
1150 if (StoreInstrCache.find(BlockPair) != StoreInstrCache.end())
1152 return I->mayAlias(AA,
MI,
false);
1155 bool SawStore =
false;
1156 bool HasAliasedStore =
false;
1165 if (BB == To || BB ==
From)
1169 if (HandledBlocks.
count(BB))
1172 HandledBlocks.
insert(BB);
1175 if (!HandledDomBlocks.
count(BB))
1176 HandledDomBlocks.
insert(BB);
1182 for (
auto *DomBB : HandledDomBlocks) {
1183 if (DomBB != BB && DT->
dominates(DomBB, BB))
1184 HasStoreCache[std::make_pair(DomBB, To)] =
true;
1185 else if(DomBB != BB && DT->
dominates(BB, DomBB))
1186 HasStoreCache[std::make_pair(
From, DomBB)] =
true;
1188 HasStoreCache[BlockPair] =
true;
1195 if (
I.isCall() ||
I.hasOrderedMemoryRef()) {
1196 for (
auto *DomBB : HandledDomBlocks) {
1197 if (DomBB != BB && DT->
dominates(DomBB, BB))
1198 HasStoreCache[std::make_pair(DomBB, To)] =
true;
1199 else if(DomBB != BB && DT->
dominates(BB, DomBB))
1200 HasStoreCache[std::make_pair(
From, DomBB)] =
true;
1202 HasStoreCache[BlockPair] =
true;
1212 if (
I.mayAlias(AA,
MI,
false))
1213 HasAliasedStore =
true;
1214 StoreInstrCache[BlockPair].push_back(&
I);
1221 HasStoreCache[BlockPair] =
false;
1222 return HasAliasedStore;
1231 assert(Preheader &&
"Cycle sink needs a preheader block");
1233 bool CanSink =
true;
1239 LLVM_DEBUG(
dbgs() <<
"CycleSink: Use not in cycle, can't sink.\n");
1254 SinkBlock =
MI.getParent();
1261 LLVM_DEBUG(
dbgs() <<
"CycleSink: Can't find nearest dominator\n");
1265 LLVM_DEBUG(
dbgs() <<
"CycleSink: Setting nearest common dom block: " <<
1274 LLVM_DEBUG(
dbgs() <<
"CycleSink: Not sinking, can't find sink block.\n");
1277 if (SinkBlock == Preheader) {
1279 dbgs() <<
"CycleSink: Not sinking, sink block is the preheader\n");
1284 dbgs() <<
"CycleSink: Not Sinking, block too large to analyse.\n");
1295 RegsToClearKillFlags.insert(MO.
getReg());
1300 assert(!
I.isDebugInstr() &&
"Should not sink debug inst");
1313 if (BB->
begin() == End)
1317 if (!
TII->isBasicBlockPrologue(*PI))
1319 for (
auto &MO :
MI.operands()) {
1326 if (
Reg.isPhysical() &&
1327 (
TII->isIgnorableUse(MO) || (
MRI &&
MRI->isConstantPhysReg(
Reg))))
1329 if (PI->modifiesRegister(
Reg,
TRI))
1332 if (PI->readsRegister(
Reg,
TRI))
1335 auto *DefOp = PI->findRegisterDefOperand(
Reg,
false,
true,
TRI);
1336 if (DefOp && !DefOp->isDead())
1346bool MachineSinking::SinkInstruction(
MachineInstr &
MI,
bool &SawStore,
1347 AllSuccsCache &AllSuccessors) {
1353 if (!
MI.isSafeToMove(AA, SawStore))
1358 if (
MI.isConvergent())
1374 bool BreakPHIEdge =
false;
1377 FindSuccToSinkTo(
MI, ParentBlock, BreakPHIEdge, AllSuccessors);
1390 if (
Reg == 0 || !
Reg.isPhysical())
1396 LLVM_DEBUG(
dbgs() <<
"Sink instr " <<
MI <<
"\tinto block " << *SuccToSinkTo);
1403 bool TryBreak =
false;
1405 MI.mayLoad() ? hasStoreBetween(ParentBlock, SuccToSinkTo,
MI) :
true;
1406 if (!
MI.isSafeToMove(AA, Store)) {
1407 LLVM_DEBUG(
dbgs() <<
" *** NOTE: Won't sink load along critical edge.\n");
1413 if (!TryBreak && !DT->
dominates(ParentBlock, SuccToSinkTo)) {
1419 if (!TryBreak && CI->
getCycle(SuccToSinkTo) &&
1434 PostponeSplitCriticalEdge(
MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
1437 "break critical edge\n");
1447 bool Status = PostponeSplitCriticalEdge(
MI, ParentBlock,
1448 SuccToSinkTo, BreakPHIEdge);
1451 "break critical edge\n");
1460 LLVM_DEBUG(
dbgs() <<
" *** Not sinking: prologue interference\n");
1466 for (
auto &MO :
MI.operands()) {
1476 if (
User.getInt()) {
1491 if (
MI.getMF()->getFunction().getSubprogram() &&
MI.isCopy())
1492 SalvageUnsunkDebugUsersOfCopy(
MI, SuccToSinkTo);
1503 RegsToClearKillFlags.insert(MO.
getReg());
1509void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
1520 for (
auto &MO :
MI.operands()) {
1529 if (
User.getParent() ==
MI.getParent())
1533 "DBG_VALUE user of vreg, but has no operand for it?");
1540 for (
auto *
User : DbgDefUsers) {
1541 for (
auto &
Reg : DbgUseRegs) {
1542 for (
auto &
DbgOp :
User->getDebugOperandsForReg(
Reg)) {
1543 DbgOp.setReg(
MI.getOperand(1).getReg());
1544 DbgOp.setSubReg(
MI.getOperand(1).getSubReg());
1601 MachineFunctionProperties::Property::NoVRegs);
1621char PostRAMachineSinking::ID = 0;
1625 "PostRA Machine Sink",
false,
false)
1640 for (
auto *
SI : SinkableBBs) {
1641 if (aliasWithRegsInLiveIn(*
SI,
Reg,
TRI)) {
1667 for (
auto DefReg : DefedRegsInCopy) {
1670 if (!BB || (SingleBB && SingleBB != BB))
1681 for (
auto U : UsedOpsInCopy) {
1687 if (UI.killsRegister(SrcReg,
TRI)) {
1688 UI.clearRegisterKills(SrcReg,
TRI);
1702 for (
unsigned DefReg : DefedRegsInCopy)
1705 for (
auto U : UsedOpsInCopy) {
1706 Register SrcReg =
MI->getOperand(U).getReg();
1709 Mask |= (*S).second;
1721 bool HasRegDependency =
false;
1722 for (
unsigned i = 0, e =
MI->getNumOperands(); i != e; ++i) {
1731 HasRegDependency =
true;
1740 }
else if (MO.
isUse()) {
1742 HasRegDependency =
true;
1748 return HasRegDependency;
1760 if (!
SI->livein_empty() &&
SI->pred_size() == 1)
1763 if (SinkableBBs.
empty())
1766 bool Changed =
false;
1770 ModifiedRegUnits.clear();
1771 UsedRegUnits.clear();
1772 SeenDbgInstrs.clear();
1782 if (
MI.isDebugValue() && !
MI.isDebugRef()) {
1784 bool IsValid =
true;
1790 ModifiedRegUnits, UsedRegUnits)) {
1798 MIUnits[*RI].push_back(MO.
getReg());
1802 for (
auto &RegOps : MIUnits)
1803 SeenDbgInstrs[RegOps.first].emplace_back(&
MI,
1804 std::move(RegOps.second));
1809 if (
MI.isDebugOrPseudoInstr())
1816 if (!
MI.isCopy() || !
MI.getOperand(0).isRenamable()) {
1824 ModifiedRegUnits, UsedRegUnits)) {
1830 "Unexpect SrcReg or DefReg");
1841 "Unexpected predecessor");
1847 for (
auto &MO :
MI.operands()) {
1852 for (
const auto &
MIRegs : SeenDbgInstrs.lookup(*RI)) {
1853 auto &Regs = DbgValsToSinkMap[
MIRegs.first];
1855 Regs.push_back(
Reg);
1859 auto DbgValsToSink = DbgValsToSinkMap.
takeVector();
1867 dbgs() <<
" *** Not sinking: prologue interference\n");
1878 ++NumPostRACopySink;
1887 bool Changed =
false;
1891 ModifiedRegUnits.init(*
TRI);
1892 UsedRegUnits.init(*
TRI);
1894 Changed |= tryToSinkCopy(BB, MF,
TRI,
TII);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
BlockVerifier::State From
This file defines the DenseSet and SmallDenseSet classes.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
const HexagonInstrInfo * TII
iv Induction Variable Users
static cl::opt< unsigned > SinkLoadInstsPerBlockThreshold("machine-sink-load-instrs-threshold", cl::desc("Do not try to find alias store for a load if there is a in-path " "block whose instruction number is higher than this threshold."), cl::init(2000), cl::Hidden)
static void clearKillFlags(MachineInstr *MI, MachineBasicBlock &CurBB, SmallVectorImpl< unsigned > &UsedOpsInCopy, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
unsigned const TargetRegisterInfo * TRI
static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo, MachineBasicBlock::iterator InsertPos, ArrayRef< MIRegs > DbgValuesToSink)
Sink an instruction and its associated debug instructions.
static cl::opt< bool > SplitEdges("machine-sink-split", cl::desc("Split critical edges during machine sinking"), cl::init(true), cl::Hidden)
static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI)
Return true if this machine instruction loads from global offset table or constant pool.
static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
Return true if MI is likely to be usable as a memory operation by the implicit null check optimizatio...
static cl::opt< bool > SinkInstsIntoCycle("sink-insts-to-avoid-spills", cl::desc("Sink instructions into cycles to avoid " "register spills"), cl::init(false), cl::Hidden)
static cl::opt< unsigned > SinkLoadBlocksThreshold("machine-sink-load-blocks-threshold", cl::desc("Do not try to find alias store for a load if the block number in " "the straight line is higher than this threshold."), cl::init(20), cl::Hidden)
static bool hasRegisterDependency(MachineInstr *MI, SmallVectorImpl< unsigned > &UsedOpsInCopy, SmallVectorImpl< unsigned > &DefedRegsInCopy, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits)
static void updateLiveIn(MachineInstr *MI, MachineBasicBlock *SuccBB, SmallVectorImpl< unsigned > &UsedOpsInCopy, SmallVectorImpl< unsigned > &DefedRegsInCopy)
static bool blockPrologueInterferes(MachineBasicBlock *BB, MachineBasicBlock::iterator End, MachineInstr &MI, const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, const MachineRegisterInfo *MRI)
Return true if a target defined block prologue instruction interferes with a sink candidate.
static cl::opt< unsigned > SplitEdgeProbabilityThreshold("machine-sink-split-probability-threshold", cl::desc("Percentage threshold for splitting single-instruction critical edge. " "If the branch threshold is higher than this threshold, we allow " "speculative execution of up to 1 instruction to avoid branching to " "splitted critical edge"), cl::init(40), cl::Hidden)
static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI, Register Reg)
If the sunk instruction is a copy, try to forward the copy instead of leaving an 'undef' DBG_VALUE in...
static MachineBasicBlock * getSingleLiveInSuccBB(MachineBasicBlock &CurBB, const SmallPtrSetImpl< MachineBasicBlock * > &SinkableBBs, unsigned Reg, const TargetRegisterInfo *TRI)
static cl::opt< unsigned > SinkIntoCycleLimit("machine-sink-cycle-limit", cl::desc("The maximum number of instructions considered for cycle sinking."), cl::init(50), cl::Hidden)
static cl::opt< bool > UseBlockFreqInfo("machine-sink-bfi", cl::desc("Use block frequency info to find successors to sink"), cl::init(true), cl::Hidden)
std::pair< MachineInstr *, SmallVector< unsigned, 2 > > MIRegs
This file implements a map that provides insertion order iteration.
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
This file defines the PointerIntPair class.
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
static bool ProcessBlock(BasicBlock &BB, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
static bool SinkInstruction(Instruction *Inst, SmallPtrSetImpl< Instruction * > &Stores, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
SinkInstruction - Determine whether it is safe to sink the specified machine instruction out of its c...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static const DILocation * getMergedLocation(const DILocation *LocA, const DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
Identifies a unique instance of a variable.
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Implements a dense probed hash-table based set.
Base class for the actual dominator tree node.
iterator_range< iterator > children()
const_toplevel_iterator toplevel_end() const
unsigned getCycleDepth(const BlockT *Block) const
get the depth for the cycle which containing a given block.
const_toplevel_iterator toplevel_begin() const
CycleT * getCycle(const BlockT *Block) const
Find the innermost cycle containing a given block.
A possibly irreducible generalization of a Loop.
BlockT * getHeader() const
bool isReducible() const
Whether the cycle is a natural loop.
BlockT * getCyclePreheader() const
Return the preheader block for this cycle.
bool contains(const BlockT *Block) const
Return whether Block is contained in the cycle.
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool shouldSink(const MachineInstr &MI) const override
A set of register units used to track register liveness.
static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void addLiveIns(const MachineBasicBlock &MBB)
Adds registers living into block MBB.
MCRegUnitMaskIterator enumerates a list of register units and their associated lane masks for Reg.
bool isValid() const
Returns true if this iterator is not yet at the end.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
MCSubRegIterator enumerates all sub-registers of Reg.
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
bool isEHPad() const
Returns true if the block is a landing pad.
instr_iterator instr_begin()
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator SkipPHIsAndLabels(iterator I)
Return the first instruction in MBB after I that is not a PHI or a label.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
unsigned succ_size() const
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
pred_iterator pred_begin()
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Remove the specified register from the live in set.
bool sizeWithoutDebugLargerThan(unsigned Limit) const
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const
getblockFreq - Return block frequency.
void onEdgeSplit(const MachineBasicBlock &NewPredecessor, const MachineBasicBlock &NewSuccessor, const MachineBranchProbabilityInfo &MBPI)
incrementally calculate block frequencies when we split edges, to avoid full CFG traversal.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
Legacy analysis pass which computes a MachineCycleInfo.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B.
bool isReachableFromEntry(const MachineBasicBlock *A)
isReachableFromEntry - Return true if A is dominated by the entry block of the function containing it...
MachineDomTreeNode * getNode(MachineBasicBlock *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
Representation of each machine instruction.
static iterator_range< filter_iterator< Operand *, std::function< bool(Operand &Op)> > > getDebugOperandsForReg(Instruction *MI, Register Reg)
Returns a range of all of the operands that correspond to a debug use of Reg.
bool hasDebugOperandForReg(Register Reg) const
Returns whether this debug value has at least one debug operand with the register Reg.
void setDebugValueUndef()
Sets all register debug operands in this debug value instruction to be undef.
const MachineBasicBlock * getParent() const
bool isCopyLike() const
Return true if the instruction behaves like a copy.
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
VectorType takeVector()
Clear the MapVector and return the underlying vector.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
virtual void releaseMemory()
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
PointerIntPair - This class implements a pair of a pointer and small integer.
Special value supplied for machine level alias analysis.
Track the current register pressure at some position in the instruction stream, and remember the high...
void runOnMachineFunction(const MachineFunction &MF)
runOnFunction - Prepare to answer questions about MF.
List of registers defined and used by a machine instruction.
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
Wrapper class representing virtual and physical registers.
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
A vector that has set insertion semantics.
void clear()
Completely clear the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I)
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
char & PostRAMachineSinkingID
This pass perform post-ra machine sink for COPY instructions.
void initializeMachineSinkingPass(PassRegistry &)
iterator_range< df_iterator< T > > depth_first(const T &G)
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
TODO: Might pack better if we changed this to a Struct of Arrays, since MachineOperand is width 32,...
static constexpr LaneBitmask getAll()
RegisterPressure computed within a region of instructions delimited by TopPos and BottomPos.
Represents a predicate at the MachineFunction level.