27 #define DEBUG_TYPE "regbankselect"
34 "Run the Fast mode (default mapping)"),
35 clEnumValN(RegBankSelect::Mode::Greedy,
"regbankselect-greedy",
36 "Use the Greedy mode (best local mapping)")));
40 "Assign register bank of generic virtual registers",
46 "Assign register bank of generic virtual registers",
false,
49 RegBankSelect::RegBankSelect(
Mode RunningMode)
51 MBFI(
nullptr), MBPI(
nullptr), OptMode(RunningMode) {
56 DEBUG(
dbgs() <<
"RegBankSelect mode overrided by command line\n");
62 assert(RBI &&
"Cannot work without RegisterBankInfo");
65 TPC = &getAnalysis<TargetPassConfig>();
67 MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
68 MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
87 bool RegBankSelect::assignmentMatch(
89 bool &OnlyAssign)
const {
101 OnlyAssign = CurRegBank ==
nullptr;
102 DEBUG(
dbgs() <<
"Does assignment already match: ";
103 if (CurRegBank)
dbgs() << *CurRegBank;
else dbgs() <<
"none";
104 dbgs() <<
" against ";
105 assert(DesiredRegBrank &&
"The mapping must be valid");
106 dbgs() << *DesiredRegBrank <<
'\n';);
107 return CurRegBank == DesiredRegBrank;
110 bool RegBankSelect::repairReg(
118 assert(NewVRegs.begin() != NewVRegs.end() &&
"We should not have to repair");
122 unsigned Src = MO.
getReg();
123 unsigned Dst = *NewVRegs.begin();
132 "We are about to create several defs for Dst");
142 std::unique_ptr<MachineInstr *[]> NewInstrs(
146 for (
const std::unique_ptr<InsertPoint> &InsertPt : RepairPt) {
152 InsertPt->insert(*CurMI);
153 NewInstrs[Idx++] = CurMI;
161 uint64_t RegBankSelect::getRepairCost(
164 assert(MO.
isReg() &&
"We should only repair register operand");
171 assert((!IsSameNumOfValues || CurRegBank) &&
"We should not have to repair");
182 if (IsSameNumOfValues) {
200 RBI->
copyCost(*DesiredRegBrank, *CurRegBank,
203 if (Cost != UINT_MAX)
206 "Legalization not available yet");
210 "Complex repairing not implemented yet");
218 "Do not know how to map this instruction");
221 MappingCost Cost = MappingCost::ImpossibleCost();
224 MappingCost CurCost = computeMapping(MI, CurMapping, LocalRepairPts, &Cost);
225 if (CurCost < Cost) {
226 DEBUG(
dbgs() <<
"New best: " << CurCost <<
'\n');
228 BestMapping = &CurMapping;
230 for (RepairingPlacement &RepairPt : LocalRepairPts)
238 BestMapping = &(*PossibleMappings.begin());
242 assert(BestMapping &&
"No suitable mapping for instruction");
246 void RegBankSelect::tryAvoidingSplit(
250 assert(RepairPt.
hasSplit() &&
"We should not have to adjust for split");
256 "Repairing placement does not match operand");
268 "Need to split for the first terminator?!");
275 RepairPt.
switchTo(RepairingPlacement::RepairingKind::Reassign);
290 "This code is for the def of a terminator");
324 unsigned Reg = MO.
getReg();
338 "Do not know which outgoing edges are relevant");
341 "Do not know where each terminator ends up");
355 assert(0 &&
"Repairing cost may not be accurate");
360 RepairPt.
switchTo(RepairingPlacement::RepairingKind::Impossible);
365 RegBankSelect::MappingCost RegBankSelect::computeMapping(
368 const RegBankSelect::MappingCost *BestCost) {
369 assert((MBFI || !BestCost) &&
"Costs comparison require MBFI");
372 return MappingCost::ImpossibleCost();
376 bool Saturated = Cost.addLocalCost(InstrMapping.
getCost());
377 assert(!Saturated &&
"Possible mapping saturated the cost");
378 DEBUG(
dbgs() <<
"Evaluating mapping cost for: " << MI);
379 DEBUG(
dbgs() <<
"With: " << InstrMapping <<
'\n');
381 if (BestCost && Cost > *BestCost) {
382 DEBUG(
dbgs() <<
"Mapping is too expensive from the start\n");
390 for (
unsigned OpIdx = 0, EndOpIdx = InstrMapping.
getNumOperands();
391 OpIdx != EndOpIdx; ++OpIdx) {
395 unsigned Reg = MO.
getReg();
400 InstrMapping.getOperandMapping(OpIdx);
403 if (assignmentMatch(Reg, ValMapping, Assign)) {
404 DEBUG(
dbgs() <<
"=> is free (match).\n");
408 DEBUG(
dbgs() <<
"=> is free (simple assignment).\n");
409 RepairPts.
emplace_back(RepairingPlacement(MI, OpIdx, *TRI, *
this,
417 RepairingPlacement &RepairPt = RepairPts.
back();
422 if (RepairPt.hasSplit())
423 tryAvoidingSplit(RepairPt, MO, ValMapping);
426 if (!RepairPt.canMaterialize()) {
427 DEBUG(
dbgs() <<
"Mapping involves impossible repairing\n");
428 return MappingCost::ImpossibleCost();
433 if (!BestCost || Saturated)
438 assert(MBFI && MBPI &&
"Cost computation requires MBFI and MBPI");
450 uint64_t RepairCost = getRepairCost(MO, ValMapping);
452 const uint64_t PercentageForBias = 5;
453 uint64_t Bias = (RepairCost * PercentageForBias + 99) / 100;
458 assert(((RepairCost < RepairCost * PercentageForBias) &&
459 (RepairCost * PercentageForBias <
460 RepairCost * PercentageForBias + 99)) &&
461 "Repairing involves more than a billion of instructions?!");
462 for (
const std::unique_ptr<InsertPoint> &InsertPt : RepairPt) {
463 assert(InsertPt->canMaterialize() &&
"We should not have made it here");
465 if (!InsertPt->isSplit())
466 Saturated = Cost.addLocalCost(RepairCost);
468 uint64_t CostForInsertPt = RepairCost;
471 assert(CostForInsertPt + Bias > CostForInsertPt &&
472 "Repairing + split bias overflows");
473 CostForInsertPt += Bias;
474 uint64_t PtCost = InsertPt->frequency(*
this) * CostForInsertPt;
476 if ((Saturated = PtCost < CostForInsertPt))
479 Saturated = Cost.addNonLocalCost(PtCost);
484 if (BestCost && Cost > *BestCost) {
485 DEBUG(
dbgs() <<
"Mapping is too expensive, stop processing\n");
495 DEBUG(
dbgs() <<
"Total cost is: " << Cost <<
"\n");
499 bool RegBankSelect::applyMapping(
506 for (RepairingPlacement &RepairPt : RepairPts) {
507 if (!RepairPt.canMaterialize() ||
511 "This should not make its way in the list");
512 unsigned OpIdx = RepairPt.getOpIdx();
515 InstrMapping.getOperandMapping(OpIdx);
516 unsigned Reg = MO.
getReg();
518 switch (RepairPt.getKind()) {
521 "Reassignment should only be for simple mapping");
525 OpdMapper.createVRegs(OpIdx);
526 if (!repairReg(MO, ValMapping, RepairPt, OpdMapper.getVRegs(OpIdx)))
534 DEBUG(
dbgs() <<
"Actual mapping of the operands: " << OpdMapper <<
'\n');
535 RBI->applyMapping(OpdMapper);
547 MappingCost DefaultCost = computeMapping(MI, BestMapping, RepairPts);
549 if (DefaultCost == MappingCost::ImpossibleCost())
553 RBI->getInstrPossibleMappings(MI);
554 if (PossibleMappings.
empty())
556 BestMapping = std::move(findBestMapping(MI, PossibleMappings, RepairPts));
559 assert(BestMapping.
verify(MI) &&
"Invalid instruction mapping");
561 DEBUG(
dbgs() <<
"Best Mapping: " << BestMapping <<
'\n');
565 return applyMapping(MI, BestMapping, RepairPts);
576 Mode SaveOptMode = OptMode;
577 if (F->hasFnAttribute(Attribute::OptimizeNone))
592 MF.getProperties().set(
596 std::string ErrStorage;
598 Err <<
"Instruction is not legal: " << MI <<
'\n';
624 if (!assignInstr(MI)) {
632 OptMode = SaveOptMode;
649 assert(MO.
isReg() &&
"Trying to repair a non-reg operand");
651 if (Kind != RepairingKind::Insert)
655 bool Before = !MO.
isDef();
681 unsigned Reg = MO.
getReg();
683 for (
auto Begin = Pred.
begin(); It != Begin && It->isTerminator(); --It)
684 if (It->modifiesRegister(Reg, &TRI)) {
695 if (It == Pred.
end())
703 unsigned Reg = MO.
getReg();
708 --It != Begin && It->isTerminator();)
709 if (It->modifiesRegister(Reg, &TRI)) {
722 assert(It->modifiesRegister(Reg, &TRI) &&
"Do not know where to split");
749 InsertPoints.emplace_back(&Point);
758 "Splitting before phis requires more points");
760 "Splitting between phis does not make sense");
763 void RegBankSelect::InstrInsertPoint::materialize() {
788 return Instr.isTerminator();
791 return Instr.getPrevNode() && Instr.getPrevNode()->isTerminator();
801 return MBFI->
getBlockFreq(Instr.getParent()).getFrequency();
812 void RegBankSelect::EdgeInsertPoint::materialize() {
817 assert(Src.isSuccessor(DstOrSplit) && DstOrSplit->isPredecessor(&Src) &&
818 "This point has already been split");
820 assert(NewBB &&
"Invalid call to materialize");
846 assert(Src.succ_size() > 1 && DstOrSplit->pred_size() > 1 &&
847 "Edge is not critical");
848 return Src.canSplitCriticalEdge(DstOrSplit);
851 RegBankSelect::MappingCost::MappingCost(
const BlockFrequency &LocalFreq)
852 : LocalCost(0), NonLocalCost(0), LocalFreq(LocalFreq.getFrequency()) {}
854 bool RegBankSelect::MappingCost::addLocalCost(uint64_t Cost) {
856 if (LocalCost + Cost < LocalCost) {
861 return isSaturated();
864 bool RegBankSelect::MappingCost::addNonLocalCost(uint64_t Cost) {
866 if (NonLocalCost + Cost < NonLocalCost) {
870 NonLocalCost += Cost;
871 return isSaturated();
874 bool RegBankSelect::MappingCost::isSaturated()
const {
875 return LocalCost == UINT64_MAX - 1 && NonLocalCost == UINT64_MAX &&
876 LocalFreq == UINT64_MAX;
879 void RegBankSelect::MappingCost::saturate() {
880 *
this = ImpossibleCost();
884 RegBankSelect::MappingCost RegBankSelect::MappingCost::ImpossibleCost() {
885 return MappingCost(UINT64_MAX, UINT64_MAX, UINT64_MAX);
894 if ((*
this == ImpossibleCost()) || (Cost == ImpossibleCost()))
895 return (*
this == ImpossibleCost()) < (Cost == ImpossibleCost());
898 if (isSaturated() || Cost.isSaturated())
899 return isSaturated() < Cost.isSaturated();
906 uint64_t ThisLocalAdjust;
907 uint64_t OtherLocalAdjust;
912 if (NonLocalCost == Cost.NonLocalCost)
915 return LocalCost < Cost.LocalCost;
920 OtherLocalAdjust = 0;
921 if (LocalCost < Cost.LocalCost)
922 OtherLocalAdjust = Cost.LocalCost - LocalCost;
924 ThisLocalAdjust = LocalCost - Cost.LocalCost;
927 ThisLocalAdjust = LocalCost;
928 OtherLocalAdjust = Cost.LocalCost;
932 uint64_t ThisNonLocalAdjust = 0;
933 uint64_t OtherNonLocalAdjust = 0;
934 if (NonLocalCost < Cost.NonLocalCost)
935 OtherNonLocalAdjust = Cost.NonLocalCost - NonLocalCost;
937 ThisNonLocalAdjust = NonLocalCost - Cost.NonLocalCost;
939 uint64_t ThisScaledCost = ThisLocalAdjust * LocalFreq;
941 bool ThisOverflows = ThisLocalAdjust && (ThisScaledCost < ThisLocalAdjust ||
942 ThisScaledCost < LocalFreq);
943 uint64_t OtherScaledCost = OtherLocalAdjust * Cost.LocalFreq;
945 bool OtherOverflows =
947 (OtherScaledCost < OtherLocalAdjust || OtherScaledCost < Cost.LocalFreq);
949 ThisOverflows |= ThisNonLocalAdjust &&
950 ThisScaledCost + ThisNonLocalAdjust < ThisNonLocalAdjust;
951 ThisScaledCost += ThisNonLocalAdjust;
952 OtherOverflows |= OtherNonLocalAdjust &&
953 OtherScaledCost + OtherNonLocalAdjust < OtherNonLocalAdjust;
954 OtherScaledCost += OtherNonLocalAdjust;
957 if (ThisOverflows && OtherOverflows)
960 if (ThisOverflows || OtherOverflows)
961 return ThisOverflows < OtherOverflows;
963 return ThisScaledCost < OtherScaledCost;
967 return LocalCost == Cost.LocalCost && NonLocalCost == Cost.NonLocalCost &&
968 LocalFreq == Cost.LocalFreq;
976 void RegBankSelect::MappingCost::print(
raw_ostream &OS)
const {
977 if (*
this == ImpossibleCost()) {
985 OS << LocalFreq <<
" * " << LocalCost <<
" + " << NonLocalCost;
Pass interface - Implemented by all 'passes'.
virtual unsigned copyCost(const RegisterBank &A, const RegisterBank &B, unsigned Size) const
Get the cost of a copy from B to A, or put differently, get the cost of A = COPY B.
bool canMaterialize() const override
Check whether this insertion point can be materialized.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool hasProperty(Property P) const
MachineBasicBlock * getMBB() const
#define LLVM_LIKELY(EXPR)
Helper class that represents how the value of an instruction may be mapped and what is the related co...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Reparing code needs to happen before InsertPoints.
static bool isTargetSpecificOpcode(unsigned Opcode)
Check whether the given Opcode is a target-specific opcode.
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
static void dump(StringRef Title, SpillInfo const &Spills)
Helper class used to get/create the virtual registers that will be used to replace the MachineOperand...
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
void setRegBank(unsigned Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
Mode
List of the modes supported by the RegBankSelect pass.
void switchTo(RepairingKind NewKind)
Change the type of this repairing placement to NewKind.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const
getblockFreq - Return block frequency.
iterator_range< succ_iterator > successors()
const PartialMapping * BreakDown
How the value is broken down between the different register banks.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Mark this repairing placement as impossible.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
AnalysisType * getAnalysisIfAvailable() const
getAnalysisIfAvailable<AnalysisType>() - Subclasses use this function to get analysis information tha...
virtual const RegisterBankInfo * getRegBankInfo() const
If the information for the register banks is available, return it.
Abstract class used to represent an insertion point in a CFG.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Reg
All possible values of the reg field in the ModR/M byte.
RepairingKind
Define the kind of action this repairing needs.
virtual bool isGlobalISelAbortEnabled() const
Check whether or not GlobalISel should abort on error.
Target-Independent Code Generator Pass Configuration Options.
LLVM_NODISCARD bool empty() const
unsigned getOpIdx() const
virtual bool isSplit() const
Does this point involve splitting an edge or block? As soon as ::getPoint is called and thus...
Function Alias Analysis false
virtual InstructionMapping getInstrMapping(const MachineInstr &MI) const
Get the mapping of the different operands of MI on the register bank.
MachineFunction & getMF()
Getter for the function we currently build.
virtual bool canMaterialize() const
Check whether this insertion point can be materialized.
iterator getLastNonDebugInstr()
Returns an iterator to the last non-debug instruction in the basic block, or end().
Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubRegIdx=0)
Prints virtual and physical registers with or without a TRI instance.
RepairingPlacement(MachineInstr &MI, unsigned OpIdx, const TargetRegisterInfo &TRI, Pass &P, RepairingKind Kind=RepairingKind::Insert)
Create a repairing placement for the OpIdx-th operand of MI.
const RegisterBank * RegBank
Register bank where the partial value lives.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
uint64_t frequency(const Pass &P) const override
Frequency of the insertion point.
ValuesClass values(OptsTy...Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
This pass implements the reg bank selector pass used in the GlobalISel pipeline.
Insertion point on an edge.
static unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI)
Get the size in bits of Reg.
const MachineOperand & getOperand(unsigned i) const
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
unsigned getNumInsertPoints() const
void initializeRegBankSelectPass(PassRegistry &)
void setMF(MachineFunction &)
Setters for the insertion point.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
Struct used to represent the placement of a repairing point for a given operand.
static cl::opt< RegBankSelect::Mode > RegBankSelectMode(cl::desc("Mode of the RegBankSelect pass"), cl::Hidden, cl::Optional, cl::values(clEnumValN(RegBankSelect::Mode::Fast,"regbankselect-fast","Run the Fast mode (default mapping)"), clEnumValN(RegBankSelect::Mode::Greedy,"regbankselect-greedy","Use the Greedy mode (best local mapping)")))
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
CloneMachineInstr - Create a new MachineInstr which is a copy of the 'Orig' instruction, identical in all ways except the instruction has no parent, prev, or next.
RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
MachineInstrBuilder buildCopy(unsigned Res, unsigned Op)
Build and insert Res<def> = COPY Op.
This class implements the register bank concept.
Helper struct that represents how a value is mapped through different register banks.
unsigned getCost() const
Get the cost.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A range adaptor for a pair of iterators.
static bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel...
bool isValid() const
Check whether this object is valid.
INITIALIZE_PASS_BEGIN(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false)
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
void emplace_back(ArgTypes &&...Args)
Insertion point before or after an instruction.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
InstrInsertPoint(MachineInstr &Instr, bool Before=true)
Create an insertion point before (Before=true) or after Instr.
uint64_t frequency(const Pass &P) const override
Frequency of the insertion point.
unsigned getNumOperands() const
Get the number of operands.
void addInsertPoint(MachineBasicBlock &MBB, bool Beginning)
Overloaded methods to add an insertion point.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool runOnMachineFunction(MachineFunction &MF) override
Walk through MF and assign a register bank to every virtual register that are still mapped to nothing...
MachineInstr * removeFromParent()
Unlink 'this' from the containing basic block, and return it without deleting it. ...
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MachineBasicBlock * SplitCriticalEdge(MachineBasicBlock *Succ, Pass &P)
Split the critical edge from this block to the given successor block, and return the newly created bl...
bool operator<(int64_t V1, const APSInt &V2)
A raw_ostream that writes to an std::string.
unsigned NumBreakDowns
Number of partial mapping to break down this value.
bool verify(const MachineInstr &MI) const
Verifiy that this mapping makes sense for MI.
(Re)assign the register bank of the operand.
virtual const LegalizerInfo * getLegalizerInfo() const
This class implements an extremely fast bulk output stream that can only output to a stream...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual void print(raw_ostream &O, const Module *M) const
print - Print out the internal state of the pass.
bool operator==(uint64_t V1, const APInt &V2)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
uint64_t frequency(const Pass &P) const override
Frequency of the insertion point.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Insertion point at the beginning or end of a basic block.
Nothing to repair, just drop this action.
bool isUnconditionalBranch(QueryType Type=AnyInBundle) const
Return true if this is a branch which always transfers control flow to some other block...
bool isSplit() const override
Does this point involve splitting an edge or block? As soon as ::getPoint is called and thus...