48 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing"
54 cl::desc(
"Always modify dest registers regardless of color"),
61 cl::desc(
"Ignore balance information, always return "
62 "(1: Even, 2: Odd)."),
71 case AArch64::FMULSrr:
72 case AArch64::FNMULSrr:
73 case AArch64::FMULDrr:
74 case AArch64::FNMULDrr:
84 case AArch64::FMSUBSrrr:
85 case AArch64::FMADDSrrr:
86 case AArch64::FNMSUBSrrr:
87 case AArch64::FNMADDSrrr:
88 case AArch64::FMSUBDrrr:
89 case AArch64::FMADDDrrr:
90 case AArch64::FNMSUBDrrr:
91 case AArch64::FNMADDDrrr:
105 static const char *ColorNames[2] = {
"Even",
"Odd" };
129 return "A57 FP Anti-dependency breaker";
144 std::map<unsigned, Chain*> &Active,
145 std::vector<std::unique_ptr<Chain>> &AllChains);
147 std::map<unsigned, Chain*> &RegChains);
149 Chain *getAndEraseNext(
Color PreferredColor, std::vector<Chain*> &
L);
156 "AArch64 A57 FP Load-Balancing",
false,
false)
205 : StartInst(MI), LastInst(MI), KillInst(nullptr),
206 StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0),
217 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
218 "Chain: broken invariant. A Chain can only be killed after its last "
237 KillIsImmutable = Immutable;
238 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
239 "Chain: broken invariant. A Chain can only be killed after its last "
268 unsigned End = KillInst ? KillInstIdx : LastInstIdx;
269 unsigned OtherEnd = Other.
KillInst ?
282 return (getKill() && isKillImmutable()) || !getKill();
291 StartInst->print(OS,
true);
293 LastInst->print(OS,
true);
296 KillInst->print(OS,
true);
317 bool Changed =
false;
318 DEBUG(
dbgs() <<
"***** AArch64A57FPLoadBalancing *****\n");
322 RCI.runOnMachineFunction(F);
324 for (
auto &
MBB : F) {
332 bool Changed =
false;
333 DEBUG(
dbgs() <<
"Running on MBB: " << MBB <<
" - scanning instructions...\n");
340 std::map<unsigned, Chain*> ActiveChains;
341 std::vector<std::unique_ptr<Chain>> AllChains;
344 scanInstruction(&
MI, Idx++, ActiveChains, AllChains);
346 DEBUG(
dbgs() <<
"Scan complete, "<< AllChains.size() <<
" chains created.\n");
356 for (
auto &
I : AllChains)
359 for (
auto &
I : AllChains)
360 for (
auto &J : AllChains)
361 if (
I != J &&
I->rangeOverlapsWith(*J))
362 EC.unionSets(
I.get(), J.get());
363 DEBUG(
dbgs() <<
"Created " << EC.getNumClasses() <<
" disjoint sets.\n");
369 std::vector<std::vector<Chain*> > V;
370 for (
auto I = EC.begin(),
E = EC.end();
I !=
E; ++
I) {
371 std::vector<Chain*> Cs(EC.member_begin(
I), EC.member_end());
372 if (Cs.empty())
continue;
373 V.push_back(std::move(Cs));
378 std::sort(V.begin(), V.end(),
379 [](
const std::vector<Chain*> &
A,
380 const std::vector<Chain*> &
B) {
381 return A.front()->startsBefore(
B.front());
398 Changed |= colorChainSet(std::move(
I), MBB, Parity);
403 Chain *AArch64A57FPLoadBalancing::getAndEraseNext(
Color PreferredColor,
404 std::vector<Chain*> &
L) {
416 const unsigned SizeFuzz = 1;
417 unsigned MinSize = L.front()->size() - SizeFuzz;
418 for (
auto I = L.begin(),
E = L.end();
I !=
E; ++
I) {
419 if ((*I)->size() <= MinSize) {
426 if ((*I)->getPreferredColor() == PreferredColor) {
434 Chain *Ch = L.front();
439 bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV,
442 bool Changed =
false;
443 DEBUG(
dbgs() <<
"colorChainSet(): #sets=" << GV.size() <<
"\n");
454 std::sort(GV.begin(), GV.end(), [](
const Chain *G1,
const Chain *G2) {
455 if (G1->size() != G2->size())
456 return G1->size() > G2->size();
457 if (G1->requiresFixup() != G2->requiresFixup())
458 return G1->requiresFixup() > G2->requiresFixup();
460 assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) &&
461 "Starts before not total order!");
462 return G1->startsBefore(G2);
465 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
466 while (Chain *
G = getAndEraseNext(PreferredColor, GV)) {
471 C =
G->getPreferredColor();
473 DEBUG(
dbgs() <<
" - Parity=" << Parity <<
", Color="
474 << ColorNames[(
int)C] <<
"\n");
479 if (
G->requiresFixup() && C !=
G->getPreferredColor()) {
480 C =
G->getPreferredColor();
481 DEBUG(
dbgs() <<
" - " <<
G->str() <<
" - not worthwhile changing; "
482 "color remains " << ColorNames[(int)C] <<
"\n");
485 Changed |= colorChain(
G, C, MBB);
487 Parity += (C == Color::Even) ?
G->size() : -
G->size();
488 PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
494 int AArch64A57FPLoadBalancing::scavengeRegister(Chain *
G,
Color C,
502 unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass;
510 for (
auto J :
I->operands()) {
514 if (J.isReg() && J.isDef()) {
517 for (; AI.isValid(); ++AI)
518 AvailableRegs.
reset(*AI);
521 for (; AI.isValid(); ++AI)
522 assert(!AvailableRegs[*AI] &&
523 "Non-dead def should have been removed by now!");
530 auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID));
531 for (
auto Reg : Ord) {
532 if (!AvailableRegs[
Reg])
534 if (C == getColor(Reg))
541 bool AArch64A57FPLoadBalancing::colorChain(Chain *G,
Color C,
543 bool Changed =
false;
544 DEBUG(
dbgs() <<
" - colorChain(" << G->str() <<
", "
545 << ColorNames[(int)C] <<
")\n");
549 int Reg = scavengeRegister(G, C, MBB);
551 DEBUG(
dbgs() <<
"Scavenging (thus coloring) failed!\n");
554 DEBUG(
dbgs() <<
" - Scavenged register: " << TRI->getName(Reg) <<
"\n");
556 std::map<unsigned, unsigned> Substs;
558 if (!G->contains(
I) && (&
I != G->getKill() || G->isKillImmutable()))
563 std::vector<unsigned> ToErase;
564 for (
auto &U :
I.operands()) {
565 if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) {
566 unsigned OrigReg = U.getReg();
567 U.setReg(Substs[OrigReg]);
571 ToErase.push_back(OrigReg);
572 }
else if (U.isRegMask()) {
573 for (
auto J : Substs) {
574 if (U.clobbersPhysReg(J.first))
575 ToErase.push_back(J.first);
580 for (
auto J : ToErase)
584 if (&
I != G->getKill()) {
588 if (G->requiresFixup() && &
I == G->getLast())
599 assert(Substs.size() == 0 &&
"No substitutions should be left active!");
602 DEBUG(
dbgs() <<
" - Kill instruction seen.\n");
606 DEBUG(
dbgs() <<
" - Destination register not changed.\n");
611 void AArch64A57FPLoadBalancing::scanInstruction(
612 MachineInstr *
MI,
unsigned Idx, std::map<unsigned, Chain *> &ActiveChains,
613 std::vector<std::unique_ptr<Chain>> &AllChains) {
618 for (
auto &
I : MI->
uses())
619 maybeKillChain(
I, Idx, ActiveChains);
620 for (
auto &
I : MI->
defs())
621 maybeKillChain(
I, Idx, ActiveChains);
627 DEBUG(
dbgs() <<
"New chain started for register "
628 << TRI->getName(DestReg) <<
" at " << *
MI);
630 auto G = llvm::make_unique<Chain>(
MI, Idx, getColor(DestReg));
631 ActiveChains[DestReg] = G.get();
632 AllChains.push_back(std::move(G));
634 }
else if (
isMla(MI)) {
638 unsigned DestReg = MI->getOperand(0).getReg();
639 unsigned AccumReg = MI->getOperand(3).getReg();
641 maybeKillChain(MI->getOperand(1), Idx, ActiveChains);
642 maybeKillChain(MI->getOperand(2), Idx, ActiveChains);
643 if (DestReg != AccumReg)
644 maybeKillChain(MI->getOperand(0), Idx, ActiveChains);
646 if (ActiveChains.find(AccumReg) != ActiveChains.end()) {
647 DEBUG(
dbgs() <<
"Chain found for accumulator register "
648 << TRI->getName(AccumReg) <<
" in MI " << *
MI);
655 if (MI->getOperand(3).isKill()) {
657 DEBUG(
dbgs() <<
"Instruction was successfully added to chain.\n");
658 ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg));
660 if (DestReg != AccumReg) {
661 ActiveChains[DestReg] = ActiveChains[AccumReg];
662 ActiveChains.erase(AccumReg);
667 DEBUG(
dbgs() <<
"Cannot add to chain because accumulator operand wasn't "
668 <<
"marked <kill>!\n");
669 maybeKillChain(MI->getOperand(3), Idx, ActiveChains);
672 DEBUG(
dbgs() <<
"Creating new chain for dest register "
673 << TRI->getName(DestReg) <<
"\n");
674 auto G = llvm::make_unique<Chain>(
MI, Idx, getColor(DestReg));
675 ActiveChains[DestReg] = G.get();
676 AllChains.push_back(std::move(G));
682 for (
auto &
I : MI->uses())
683 maybeKillChain(
I, Idx, ActiveChains);
684 for (
auto &
I : MI->defs())
685 maybeKillChain(
I, Idx, ActiveChains);
690 void AArch64A57FPLoadBalancing::
692 std::map<unsigned, Chain*> &ActiveChains) {
700 if (MO.
isKill() && ActiveChains.find(MO.
getReg()) != ActiveChains.end()) {
703 ActiveChains[MO.
getReg()]->setKill(MI, Idx, MO.
isTied());
705 ActiveChains.erase(MO.
getReg());
709 for (
auto I = ActiveChains.begin(),
E = ActiveChains.end();
712 DEBUG(
dbgs() <<
"Kill (regmask) seen for chain "
713 << TRI->getName(
I->first) <<
"\n");
714 I->second->setKill(MI, Idx,
true);
715 ActiveChains.erase(
I++);
723 Color AArch64A57FPLoadBalancing::getColor(
unsigned Reg) {
724 if ((TRI->getEncodingValue(Reg) % 2) == 0)
732 return new AArch64A57FPLoadBalancing();
bool startsBefore(const Chain *Other) const
Return true if this chain starts before Other.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
MachineInstr * StartInst
The important (marker) instructions.
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
static bool runOnBasicBlock(BasicBlock &BB)
MachineInstrBundleIterator< MachineInstr > iterator
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const TargetRegisterInfo * getTargetRegisterInfo() const
bool balanceFPOps() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
std::string str() const
Return a simple string representation of the chain.
INITIALIZE_PASS_BEGIN(AArch64A57FPLoadBalancing, DEBUG_TYPE,"AArch64 A57 FP Load-Balancing", false, false) INITIALIZE_PASS_END(AArch64A57FPLoadBalancing
void forward()
Move the internal MBB iterator and update register states.
MachineBasicBlock::iterator begin() const
FunctionPass * createAArch64A57FPLoadBalancing()
BitVector getRegsAvailable(const TargetRegisterClass *RC)
Return all available registers in the register class in Mask.
AArch64 A57 FP Load false
bool requiresFixup() const
Return true if the group will require a fixup MOV at the end.
static cl::opt< bool > TransformAll("aarch64-a57-fp-load-balancing-force-all", cl::desc("Always modify dest registers regardless of color"), cl::init(false), cl::Hidden)
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
bool contains(MachineInstr &MI)
Return true if MI is a member of the chain.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
void initializeAArch64A57FPLoadBalancingPass(PassRegistry &)
MachineInstr * getKill() const
Return the "kill" instruction (as set with setKill()) or NULL.
initializer< Ty > init(const Ty &Val)
void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
clearBitsNotInMask - Clear a bit in this vector for every '0' bit in Mask.
This file declares the machine register scavenger class.
unsigned const MachineRegisterInfo * MRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const MachineOperand & getOperand(unsigned i) const
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
MCRegAliasIterator enumerates all registers aliasing Reg.
Represent the analysis usage information of a pass.
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
bool isKillImmutable() const
Can the Kill instruction (assuming one exists) be modified?
static const unsigned End
bool rangeOverlapsWith(const Chain &Other) const
Return true if this chain (StartInst..KillInst) overlaps with Other.
FunctionPass class - This class is used to implement most global optimizations.
AArch64 A57 FP Load Balancing
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
Color LastColor
The "color" of LastInst.
void add(MachineInstr *MI, unsigned Idx, Color C)
Add a new instruction into the chain.
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
Color
A "color", which is either even or odd.
MachineOperand class - Representation of each machine instruction operand.
unsigned size() const
Return the number of instructions in the chain.
Promote Memory to Register
static bool isMla(MachineInstr *MI)
void setPreservesCFG()
This function should be called by the pass, iff they do not:
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
MachineInstr * getLast() const
Return the last instruction in the chain.
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
std::set< MachineInstr * > Insts
All instructions in the chain.
MachineBasicBlock::iterator end() const
Return an instruction that can be used as an iterator for the end of the chain.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
Representation of each machine instruction.
void enterBasicBlock(MachineBasicBlock &MBB)
Start tracking liveness from the begin of basic block MBB.
bool KillIsImmutable
True if KillInst cannot be modified.
A Chain is a sequence of instructions that are linked together by an accumulation operand...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
unsigned getReg() const
getReg - Returns the register number.
unsigned StartInstIdx
The index, from the start of the basic block, that each marker appears.
void setKill(MachineInstr *MI, unsigned Idx, bool Immutable)
Inform the chain that its last active register (the dest register of LastInst) is killed by MI with n...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A raw_ostream that writes to an std::string.
static cl::opt< unsigned > OverrideBalance("aarch64-a57-fp-load-balancing-override", cl::desc("Ignore balance information, always return ""(1: Even, 2: Odd)."), cl::init(0), cl::Hidden)
Color getPreferredColor()
Return the preferred color of this chain.
static bool isMul(MachineInstr *MI)
StringRef - Represent a constant reference to a string, i.e.
MachineInstr * getStart() const
Return the first instruction in the chain.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
Properties which a MachineFunction may have at a given point in time.
Chain(MachineInstr *MI, unsigned Idx, Color C)