33 #define DEBUG_TYPE "x86-domain-reassignment"
35 STATISTIC(NumClosuresConverted,
"Number of closures converted by the pass");
42 enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
45 return X86::GR64RegClass.hasSubClassEq(RC) ||
46 X86::GR32RegClass.hasSubClassEq(RC) ||
47 X86::GR16RegClass.hasSubClassEq(RC) ||
48 X86::GR8RegClass.hasSubClassEq(RC);
53 return X86::VK16RegClass.hasSubClassEq(RC);
69 if (X86::GR8RegClass.hasSubClassEq(SrcRC))
70 return &X86::VK8RegClass;
71 if (X86::GR16RegClass.hasSubClassEq(SrcRC))
72 return &X86::VK16RegClass;
73 if (X86::GR32RegClass.hasSubClassEq(SrcRC))
74 return &X86::VK32RegClass;
75 if (X86::GR64RegClass.hasSubClassEq(SrcRC))
76 return &X86::VK64RegClass;
82 class InstrConverterBase {
87 InstrConverterBase(
unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
89 virtual ~InstrConverterBase() =
default;
94 assert(
MI->getOpcode() == SrcOpcode &&
95 "Wrong instruction passed to converter");
113 class InstrIgnore :
public InstrConverterBase {
115 InstrIgnore(
unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
119 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
130 class InstrReplacer :
public InstrConverterBase {
135 InstrReplacer(
unsigned SrcOpcode,
unsigned DstOpcode)
136 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
140 if (!InstrConverterBase::isLegal(
MI,
TII))
144 for (
const auto &MO :
MI->implicit_operands())
145 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
146 !
TII->get(DstOpcode).hasImplicitDefOfPhysReg(MO.getReg()))
153 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
158 for (
auto &
Op :
MI->explicit_operands())
172 class InstrReplacerDstCOPY :
public InstrConverterBase {
176 InstrReplacerDstCOPY(
unsigned SrcOpcode,
unsigned DstOpcode)
177 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
181 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
193 .
add(
MI->getOperand(0))
208 class InstrCOPYReplacer :
public InstrReplacer {
212 InstrCOPYReplacer(
unsigned SrcOpcode, RegDomain DstDomain,
unsigned DstOpcode)
213 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
217 if (!InstrConverterBase::isLegal(
MI,
TII))
223 if (DstReg.
isPhysical() && (X86::GR8RegClass.contains(DstReg) ||
224 X86::GR16RegClass.contains(DstReg)))
227 if (SrcReg.
isPhysical() && (X86::GR8RegClass.contains(SrcReg) ||
228 X86::GR16RegClass.contains(SrcReg)))
236 assert(
MI->getOpcode() == TargetOpcode::COPY &&
"Expected a COPY");
238 for (
const auto &MO :
MI->operands()) {
249 if (OpDomain == DstDomain)
257 class InstrReplaceWithCopy :
public InstrConverterBase {
262 InstrReplaceWithCopy(
unsigned SrcOpcode,
unsigned SrcOpIdx)
263 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
267 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
269 TII->get(TargetOpcode::COPY))
270 .
add({
MI->getOperand(0),
MI->getOperand(SrcOpIdx)});
282 typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
285 InstrConverterBaseMap;
307 std::bitset<NumDomains> LegalDstDomains;
314 Closure(
unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) :
ID(
ID) {
315 for (RegDomain
D : LegalDstDomainList)
316 LegalDstDomains.set(
D);
320 void setAllIllegal() { LegalDstDomains.reset(); }
323 bool hasLegalDstDomain()
const {
return LegalDstDomains.any(); }
326 bool isLegal(RegDomain RD)
const {
return LegalDstDomains[RD]; }
329 void setIllegal(RegDomain RD) { LegalDstDomains[RD] =
false; }
349 dbgs() <<
"Registers: ";
357 dbgs() <<
"\n" <<
"Instructions:";
365 unsigned getID()
const {
395 return "X86 Domain Reassignment Pass";
400 InstrConverterBaseMap Converters;
403 void initConverters();
413 void reassign(
const Closure &
C, RegDomain
Domain)
const;
419 bool isReassignmentProfitable(
const Closure &
C, RegDomain
Domain)
const;
422 double calculateCost(
const Closure &
C, RegDomain
Domain)
const;
429 void X86DomainReassignment::visitRegister(Closure &
C,
Register Reg,
432 if (EnclosedEdges.count(
Reg))
435 if (!
Reg.isVirtual())
449 Worklist.push_back(
Reg);
452 void X86DomainReassignment::encloseInstr(Closure &
C,
MachineInstr *
MI) {
453 auto I = EnclosedInstrs.find(
MI);
454 if (
I != EnclosedInstrs.end()) {
455 if (
I->second !=
C.getID())
462 EnclosedInstrs[
MI] =
C.getID();
463 C.addInstruction(
MI);
468 for (
int i = 0;
i != NumDomains; ++
i) {
469 if (
C.isLegal((RegDomain)
i)) {
470 auto I = Converters.find({
i,
MI->getOpcode()});
471 if (
I == Converters.end() || !
I->second->isLegal(
MI,
TII))
472 C.setIllegal((RegDomain)
i);
477 double X86DomainReassignment::calculateCost(
const Closure &
C,
478 RegDomain DstDomain)
const {
479 assert(
C.isLegal(DstDomain) &&
"Cannot calculate cost for illegal closure");
482 for (
auto *
MI :
C.instructions())
483 Cost += Converters.find({DstDomain,
MI->getOpcode()})
484 ->second->getExtraCost(
MI,
MRI);
488 bool X86DomainReassignment::isReassignmentProfitable(
const Closure &
C,
490 return calculateCost(
C,
Domain) < 0.0;
493 void X86DomainReassignment::reassign(
const Closure &
C, RegDomain
Domain)
const {
494 assert(
C.isLegal(
Domain) &&
"Cannot convert illegal closure");
499 for (
auto *
MI :
C.instructions())
500 if (Converters.find({Domain, MI->getOpcode()})
501 ->second->convertInstr(
MI,
TII,
MRI))
502 ToErase.push_back(
MI);
516 for (
auto *
MI : ToErase)
517 MI->eraseFromParent();
524 if (!
MI.mayLoadOrStore())
529 if (MemOpStart == -1)
533 for (
unsigned MemOpIdx = MemOpStart,
535 MemOpIdx < MemOpEnd; ++MemOpIdx) {
537 if (
Op.isReg() &&
Op.getReg() ==
Reg)
543 void X86DomainReassignment::buildClosure(Closure &
C,
Register Reg) {
545 RegDomain
Domain = NoDomain;
547 while (!Worklist.empty()) {
551 if (!
C.insertEdge(CurReg))
553 EnclosedEdges.insert(
Reg);
566 for (
int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
567 if (OpIdx ==
MemOp) {
573 if (!
Op.isReg() || !
Op.isUse())
575 visitRegister(
C,
Op.getReg(),
Domain, Worklist);
588 for (
auto &DefOp :
UseMI.defs()) {
597 visitRegister(
C, DefReg,
Domain, Worklist);
603 void X86DomainReassignment::initConverters() {
604 Converters[{MaskDomain, TargetOpcode::PHI}] =
605 std::make_unique<InstrIgnore>(TargetOpcode::PHI);
607 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
608 std::make_unique<InstrIgnore>(TargetOpcode::IMPLICIT_DEF);
610 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
611 std::make_unique<InstrReplaceWithCopy>(TargetOpcode::INSERT_SUBREG, 2);
613 Converters[{MaskDomain, TargetOpcode::COPY}] =
614 std::make_unique<InstrCOPYReplacer>(TargetOpcode::COPY, MaskDomain,
617 auto createReplacerDstCOPY = [&](
unsigned From,
unsigned To) {
618 Converters[{MaskDomain,
From}] =
619 std::make_unique<InstrReplacerDstCOPY>(
From, To);
622 createReplacerDstCOPY(X86::MOVZX32rm16, X86::KMOVWkm);
623 createReplacerDstCOPY(X86::MOVZX64rm16, X86::KMOVWkm);
625 createReplacerDstCOPY(X86::MOVZX32rr16, X86::KMOVWkk);
626 createReplacerDstCOPY(X86::MOVZX64rr16, X86::KMOVWkk);
629 createReplacerDstCOPY(X86::MOVZX16rm8, X86::KMOVBkm);
630 createReplacerDstCOPY(X86::MOVZX32rm8, X86::KMOVBkm);
631 createReplacerDstCOPY(X86::MOVZX64rm8, X86::KMOVBkm);
633 createReplacerDstCOPY(X86::MOVZX16rr8, X86::KMOVBkk);
634 createReplacerDstCOPY(X86::MOVZX32rr8, X86::KMOVBkk);
635 createReplacerDstCOPY(X86::MOVZX64rr8, X86::KMOVBkk);
638 auto createReplacer = [&](
unsigned From,
unsigned To) {
639 Converters[{MaskDomain,
From}] = std::make_unique<InstrReplacer>(
From, To);
642 createReplacer(X86::MOV16rm, X86::KMOVWkm);
643 createReplacer(X86::MOV16mr, X86::KMOVWmk);
644 createReplacer(X86::MOV16rr, X86::KMOVWkk);
645 createReplacer(X86::SHR16ri, X86::KSHIFTRWri);
646 createReplacer(X86::SHL16ri, X86::KSHIFTLWri);
647 createReplacer(X86::NOT16r, X86::KNOTWrr);
648 createReplacer(X86::OR16rr, X86::KORWrr);
649 createReplacer(X86::AND16rr, X86::KANDWrr);
650 createReplacer(X86::XOR16rr, X86::KXORWrr);
653 createReplacer(X86::MOV32rm, X86::KMOVDkm);
654 createReplacer(X86::MOV64rm, X86::KMOVQkm);
656 createReplacer(X86::MOV32mr, X86::KMOVDmk);
657 createReplacer(X86::MOV64mr, X86::KMOVQmk);
659 createReplacer(X86::MOV32rr, X86::KMOVDkk);
660 createReplacer(X86::MOV64rr, X86::KMOVQkk);
662 createReplacer(X86::SHR32ri, X86::KSHIFTRDri);
663 createReplacer(X86::SHR64ri, X86::KSHIFTRQri);
665 createReplacer(X86::SHL32ri, X86::KSHIFTLDri);
666 createReplacer(X86::SHL64ri, X86::KSHIFTLQri);
668 createReplacer(X86::ADD32rr, X86::KADDDrr);
669 createReplacer(X86::ADD64rr, X86::KADDQrr);
671 createReplacer(X86::NOT32r, X86::KNOTDrr);
672 createReplacer(X86::NOT64r, X86::KNOTQrr);
674 createReplacer(X86::OR32rr, X86::KORDrr);
675 createReplacer(X86::OR64rr, X86::KORQrr);
677 createReplacer(X86::AND32rr, X86::KANDDrr);
678 createReplacer(X86::AND64rr, X86::KANDQrr);
680 createReplacer(X86::ANDN32rr, X86::KANDNDrr);
681 createReplacer(X86::ANDN64rr, X86::KANDNQrr);
683 createReplacer(X86::XOR32rr, X86::KXORDrr);
684 createReplacer(X86::XOR64rr, X86::KXORQrr);
693 createReplacer(X86::ADD8rr, X86::KADDBrr);
694 createReplacer(X86::ADD16rr, X86::KADDWrr);
696 createReplacer(X86::AND8rr, X86::KANDBrr);
698 createReplacer(X86::MOV8rm, X86::KMOVBkm);
699 createReplacer(X86::MOV8mr, X86::KMOVBmk);
700 createReplacer(X86::MOV8rr, X86::KMOVBkk);
702 createReplacer(X86::NOT8r, X86::KNOTBrr);
704 createReplacer(X86::OR8rr, X86::KORBrr);
706 createReplacer(X86::SHR8ri, X86::KSHIFTRBri);
707 createReplacer(X86::SHL8ri, X86::KSHIFTLBri);
714 createReplacer(X86::XOR8rr, X86::KXORBrr);
718 bool X86DomainReassignment::runOnMachineFunction(
MachineFunction &MF) {
725 dbgs() <<
"***** Machine Function before Domain Reassignment *****\n");
734 if (!STI->hasAVX512() || !STI->hasBWI())
740 TII = STI->getInstrInfo();
742 bool Changed =
false;
744 EnclosedEdges.clear();
745 EnclosedInstrs.clear();
747 std::vector<Closure> Closures;
750 unsigned ClosureID = 0;
759 if (EnclosedEdges.count(
Reg))
763 Closure
C(ClosureID++, {MaskDomain});
764 buildClosure(
C,
Reg);
767 if (!
C.empty() &&
C.isLegal(MaskDomain))
771 for (Closure &
C : Closures) {
773 if (isReassignmentProfitable(
C, MaskDomain)) {
774 reassign(
C, MaskDomain);
775 ++NumClosuresConverted;
781 dbgs() <<
"***** Machine Function after Domain Reassignment *****\n");
788 "X86 Domain Reassignment Pass",
false,
false)
792 return new X86DomainReassignment();