34#define DEBUG_TYPE "x86-domain-reassignment"
36STATISTIC(NumClosuresConverted,
"Number of closures converted by the pass");
43enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
46 return X86::GR64RegClass.hasSubClassEq(RC) ||
47 X86::GR32RegClass.hasSubClassEq(RC) ||
48 X86::GR16RegClass.hasSubClassEq(RC) ||
49 X86::GR8RegClass.hasSubClassEq(RC);
54 return X86::VK16RegClass.hasSubClassEq(RC);
70 if (X86::GR8RegClass.hasSubClassEq(SrcRC))
71 return &X86::VK8RegClass;
72 if (X86::GR16RegClass.hasSubClassEq(SrcRC))
73 return &X86::VK16RegClass;
74 if (X86::GR32RegClass.hasSubClassEq(SrcRC))
75 return &X86::VK32RegClass;
76 if (X86::GR64RegClass.hasSubClassEq(SrcRC))
77 return &X86::VK64RegClass;
83class InstrConverterBase {
88 InstrConverterBase(
unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
90 virtual ~InstrConverterBase() =
default;
95 assert(
MI->getOpcode() == SrcOpcode &&
96 "Wrong instruction passed to converter");
114class InstrIgnore :
public InstrConverterBase {
116 InstrIgnore(
unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
120 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
131class InstrReplacer :
public InstrConverterBase {
136 InstrReplacer(
unsigned SrcOpcode,
unsigned DstOpcode)
137 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
141 if (!InstrConverterBase::isLegal(
MI,
TII))
145 for (
const auto &MO :
MI->implicit_operands())
146 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
147 !
TII->get(DstOpcode).hasImplicitDefOfPhysReg(MO.getReg()))
154 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
159 for (
auto &
Op :
MI->explicit_operands())
173class InstrReplacerDstCOPY :
public InstrConverterBase {
177 InstrReplacerDstCOPY(
unsigned SrcOpcode,
unsigned DstOpcode)
178 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
182 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
187 TII->getRegClass(
TII->get(DstOpcode), 0,
MRI->getTargetRegisterInfo(),
194 .
add(
MI->getOperand(0))
209class InstrCOPYReplacer :
public InstrReplacer {
213 InstrCOPYReplacer(
unsigned SrcOpcode, RegDomain DstDomain,
unsigned DstOpcode)
214 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
218 if (!InstrConverterBase::isLegal(
MI,
TII))
224 if (DstReg.
isPhysical() && (X86::GR8RegClass.contains(DstReg) ||
225 X86::GR16RegClass.contains(DstReg)))
228 if (SrcReg.
isPhysical() && (X86::GR8RegClass.contains(SrcReg) ||
229 X86::GR16RegClass.contains(SrcReg)))
237 assert(
MI->getOpcode() == TargetOpcode::COPY &&
"Expected a COPY");
239 for (
const auto &MO :
MI->operands()) {
243 if (MO.getReg().isPhysical())
246 RegDomain OpDomain =
getDomain(
MRI->getRegClass(MO.getReg()),
247 MRI->getTargetRegisterInfo());
250 if (OpDomain == DstDomain)
258class InstrReplaceWithCopy :
public InstrConverterBase {
263 InstrReplaceWithCopy(
unsigned SrcOpcode,
unsigned SrcOpIdx)
264 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
268 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
270 TII->get(TargetOpcode::COPY))
271 .
add({
MI->getOperand(0),
MI->getOperand(SrcOpIdx)});
283typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
286 InstrConverterBaseMap;
308 std::bitset<NumDomains> LegalDstDomains;
315 Closure(
unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) :
ID(
ID) {
316 for (RegDomain
D : LegalDstDomainList)
317 LegalDstDomains.set(
D);
321 void setAllIllegal() { LegalDstDomains.reset(); }
324 bool hasLegalDstDomain()
const {
return LegalDstDomains.any(); }
327 bool isLegal(RegDomain RD)
const {
return LegalDstDomains[RD]; }
330 void setIllegal(RegDomain RD) { LegalDstDomains[RD] =
false; }
332 bool empty()
const {
return Edges.
empty(); }
334 bool insertEdge(
Register Reg) {
return Edges.
insert(Reg).second; }
350 dbgs() <<
"Registers: ";
358 dbgs() <<
"\n" <<
"Instructions:";
366 unsigned getID()
const {
396 return "X86 Domain Reassignment Pass";
401 InstrConverterBaseMap Converters;
404 void initConverters();
407 void buildClosure(Closure &,
Register Reg);
414 void reassign(
const Closure &
C, RegDomain
Domain)
const;
420 bool isReassignmentProfitable(
const Closure &
C, RegDomain
Domain)
const;
423 double calculateCost(
const Closure &
C, RegDomain
Domain)
const;
426char X86DomainReassignment::ID = 0;
430void X86DomainReassignment::visitRegister(Closure &
C,
Register Reg,
433 if (!
Reg.isVirtual())
439 if (!
MRI->hasOneDef(Reg))
442 RegDomain RD =
getDomain(
MRI->getRegClass(Reg),
MRI->getTargetRegisterInfo());
453void X86DomainReassignment::encloseInstr(Closure &
C,
MachineInstr *
MI) {
454 auto I = EnclosedInstrs.find(
MI);
455 if (
I != EnclosedInstrs.end()) {
456 if (
I->second !=
C.getID())
463 EnclosedInstrs[
MI] =
C.getID();
464 C.addInstruction(
MI);
469 for (
int i = 0; i != NumDomains; ++i) {
470 if (
C.isLegal((RegDomain)i)) {
471 auto I = Converters.find({i,
MI->getOpcode()});
472 if (
I == Converters.end() || !
I->second->isLegal(
MI,
TII))
473 C.setIllegal((RegDomain)i);
478double X86DomainReassignment::calculateCost(
const Closure &
C,
479 RegDomain DstDomain)
const {
480 assert(
C.isLegal(DstDomain) &&
"Cannot calculate cost for illegal closure");
483 for (
auto *
MI :
C.instructions())
484 Cost += Converters.find({DstDomain,
MI->getOpcode()})
485 ->second->getExtraCost(
MI,
MRI);
489bool X86DomainReassignment::isReassignmentProfitable(
const Closure &
C,
491 return calculateCost(
C,
Domain) < 0.0;
494void X86DomainReassignment::reassign(
const Closure &
C, RegDomain
Domain)
const {
495 assert(
C.isLegal(
Domain) &&
"Cannot convert illegal closure");
500 for (
auto *
MI :
C.instructions())
501 if (Converters.find({Domain, MI->getOpcode()})
502 ->second->convertInstr(
MI,
TII,
MRI))
508 MRI->setRegClass(Reg, getDstRC(
MRI->getRegClass(Reg),
Domain));
509 for (
auto &MO :
MRI->use_operands(Reg)) {
517 for (
auto *
MI : ToErase)
518 MI->eraseFromParent();
525 if (!
MI.mayLoadOrStore())
530 if (MemOpStart == -1)
534 for (
unsigned MemOpIdx = MemOpStart,
536 MemOpIdx < MemOpEnd; ++MemOpIdx) {
538 if (
Op.isReg() &&
Op.getReg() == Reg)
544void X86DomainReassignment::buildClosure(Closure &
C,
Register Reg) {
546 RegDomain
Domain = NoDomain;
547 visitRegister(
C, Reg,
Domain, Worklist);
548 while (!Worklist.
empty()) {
552 if (!
C.insertEdge(CurReg))
567 for (
int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
568 if (OpIdx ==
MemOp) {
574 if (!
Op.isReg() || !
Op.isUse())
576 visitRegister(
C,
Op.getReg(),
Domain, Worklist);
580 for (
auto &
UseMI :
MRI->use_nodbg_instructions(CurReg)) {
589 for (
auto &DefOp :
UseMI.defs()) {
598 visitRegister(
C, DefReg,
Domain, Worklist);
604void X86DomainReassignment::initConverters() {
605 Converters[{MaskDomain, TargetOpcode::PHI}] =
606 std::make_unique<InstrIgnore>(TargetOpcode::PHI);
608 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
609 std::make_unique<InstrIgnore>(TargetOpcode::IMPLICIT_DEF);
611 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
612 std::make_unique<InstrReplaceWithCopy>(TargetOpcode::INSERT_SUBREG, 2);
614 Converters[{MaskDomain, TargetOpcode::COPY}] =
615 std::make_unique<InstrCOPYReplacer>(TargetOpcode::COPY, MaskDomain,
618 auto createReplacerDstCOPY = [&](
unsigned From,
unsigned To) {
619 Converters[{MaskDomain,
From}] =
620 std::make_unique<InstrReplacerDstCOPY>(
From, To);
623 createReplacerDstCOPY(X86::MOVZX32rm16, X86::KMOVWkm);
624 createReplacerDstCOPY(X86::MOVZX64rm16, X86::KMOVWkm);
626 createReplacerDstCOPY(X86::MOVZX32rr16, X86::KMOVWkk);
627 createReplacerDstCOPY(X86::MOVZX64rr16, X86::KMOVWkk);
630 createReplacerDstCOPY(X86::MOVZX16rm8, X86::KMOVBkm);
631 createReplacerDstCOPY(X86::MOVZX32rm8, X86::KMOVBkm);
632 createReplacerDstCOPY(X86::MOVZX64rm8, X86::KMOVBkm);
634 createReplacerDstCOPY(X86::MOVZX16rr8, X86::KMOVBkk);
635 createReplacerDstCOPY(X86::MOVZX32rr8, X86::KMOVBkk);
636 createReplacerDstCOPY(X86::MOVZX64rr8, X86::KMOVBkk);
639 auto createReplacer = [&](
unsigned From,
unsigned To) {
640 Converters[{MaskDomain,
From}] = std::make_unique<InstrReplacer>(
From, To);
643 createReplacer(X86::MOV16rm, X86::KMOVWkm);
644 createReplacer(X86::MOV16mr, X86::KMOVWmk);
645 createReplacer(X86::MOV16rr, X86::KMOVWkk);
646 createReplacer(X86::SHR16ri, X86::KSHIFTRWri);
647 createReplacer(X86::SHL16ri, X86::KSHIFTLWri);
648 createReplacer(X86::NOT16r, X86::KNOTWrr);
649 createReplacer(X86::OR16rr, X86::KORWrr);
650 createReplacer(X86::AND16rr, X86::KANDWrr);
651 createReplacer(X86::XOR16rr, X86::KXORWrr);
654 createReplacer(X86::MOV32rm, X86::KMOVDkm);
655 createReplacer(X86::MOV64rm, X86::KMOVQkm);
657 createReplacer(X86::MOV32mr, X86::KMOVDmk);
658 createReplacer(X86::MOV64mr, X86::KMOVQmk);
660 createReplacer(X86::MOV32rr, X86::KMOVDkk);
661 createReplacer(X86::MOV64rr, X86::KMOVQkk);
663 createReplacer(X86::SHR32ri, X86::KSHIFTRDri);
664 createReplacer(X86::SHR64ri, X86::KSHIFTRQri);
666 createReplacer(X86::SHL32ri, X86::KSHIFTLDri);
667 createReplacer(X86::SHL64ri, X86::KSHIFTLQri);
669 createReplacer(X86::ADD32rr, X86::KADDDrr);
670 createReplacer(X86::ADD64rr, X86::KADDQrr);
672 createReplacer(X86::NOT32r, X86::KNOTDrr);
673 createReplacer(X86::NOT64r, X86::KNOTQrr);
675 createReplacer(X86::OR32rr, X86::KORDrr);
676 createReplacer(X86::OR64rr, X86::KORQrr);
678 createReplacer(X86::AND32rr, X86::KANDDrr);
679 createReplacer(X86::AND64rr, X86::KANDQrr);
681 createReplacer(X86::ANDN32rr, X86::KANDNDrr);
682 createReplacer(X86::ANDN64rr, X86::KANDNQrr);
684 createReplacer(X86::XOR32rr, X86::KXORDrr);
685 createReplacer(X86::XOR64rr, X86::KXORQrr);
694 createReplacer(X86::ADD8rr, X86::KADDBrr);
695 createReplacer(X86::ADD16rr, X86::KADDWrr);
697 createReplacer(X86::AND8rr, X86::KANDBrr);
699 createReplacer(X86::MOV8rm, X86::KMOVBkm);
700 createReplacer(X86::MOV8mr, X86::KMOVBmk);
701 createReplacer(X86::MOV8rr, X86::KMOVBkk);
703 createReplacer(X86::NOT8r, X86::KNOTBrr);
705 createReplacer(X86::OR8rr, X86::KORBrr);
707 createReplacer(X86::SHR8ri, X86::KSHIFTRBri);
708 createReplacer(X86::SHL8ri, X86::KSHIFTLBri);
715 createReplacer(X86::XOR8rr, X86::KXORBrr);
726 dbgs() <<
"***** Machine Function before Domain Reassignment *****\n");
735 if (!STI->hasAVX512() || !STI->hasBWI())
739 assert(
MRI->isSSA() &&
"Expected MIR to be in SSA form");
741 TII = STI->getInstrInfo();
743 bool Changed =
false;
745 EnclosedEdges.clear();
746 EnclosedEdges.resize(
MRI->getNumVirtRegs());
747 EnclosedInstrs.clear();
749 std::vector<Closure> Closures;
752 unsigned ClosureID = 0;
753 for (
unsigned Idx = 0;
Idx <
MRI->getNumVirtRegs(); ++
Idx) {
757 if (!isGPR(
MRI->getRegClass(Reg)))
761 if (EnclosedEdges.test(
Idx))
765 Closure
C(ClosureID++, {MaskDomain});
766 buildClosure(
C, Reg);
769 if (!
C.empty() &&
C.isLegal(MaskDomain))
770 Closures.push_back(std::move(
C));
773 for (Closure &
C : Closures) {
775 if (isReassignmentProfitable(
C, MaskDomain)) {
776 reassign(
C, MaskDomain);
777 ++NumClosuresConverted;
783 dbgs() <<
"***** Machine Function after Domain Reassignment *****\n");
790 "X86 Domain Reassignment Pass",
false,
false)
794 return new X86DomainReassignment();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
const HexagonInstrInfo * TII
Select target instructions out of generic instructions
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static cl::opt< bool > DisableX86DomainReassignment("disable-x86-domain-reassignment", cl::Hidden, cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(false))
static bool usedAsAddr(const MachineInstr &MI, Register Reg, const TargetInstrInfo *TII)
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
Describe properties that are true of each instruction in the target description file.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
std::pair< iterator, bool > insert(const ValueT &V)
A range adaptor for a pair of iterators.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
Reg
All possible values of the reg field in the ModR/M byte.
int getMemoryOperandNo(uint64_t TSFlags)
The function returns the MCInst operand # for the first field of the memory operand.
unsigned getOperandBias(const MCInstrDesc &Desc)
Compute whether all of the def operands are repeated in the uses and therefore should be skipped.
@ AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createX86DomainReassignmentPass()
Return a Machine IR pass that reassigns instruction chains from one domain to another,...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.