46#include "llvm/IR/IntrinsicsAArch64.h"
53#define DEBUG_TYPE "aarch64-isel"
66#define GET_GLOBALISEL_PREDICATE_BITSET
67#include "AArch64GenGlobalISel.inc"
68#undef GET_GLOBALISEL_PREDICATE_BITSET
88 ProduceNonFlagSettingCondBr =
136 bool tryOptAndIntoCompareBranch(
MachineInstr &AndInst,
bool Invert,
203 bool selectVectorLoadIntrinsic(
unsigned Opc,
unsigned NumVecs,
205 bool selectVectorLoadLaneIntrinsic(
unsigned Opc,
unsigned NumVecs,
207 void selectVectorStoreIntrinsic(
MachineInstr &
I,
unsigned NumVecs,
209 bool selectVectorStoreLaneIntrinsic(
MachineInstr &
I,
unsigned NumVecs,
226 unsigned Opc1,
unsigned Opc2,
bool isExt);
232 unsigned emitConstantPoolEntry(
const Constant *CPVal,
251 std::optional<CmpInst::Predicate> = std::nullopt)
const;
254 emitInstr(
unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
255 std::initializer_list<llvm::SrcOp> SrcOps,
257 const ComplexRendererFns &RenderFns = std::nullopt)
const;
292 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
315 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
337 std::pair<MachineInstr *, AArch64CC::CondCode>
372 ComplexRendererFns selectShiftA_32(
const MachineOperand &Root)
const;
373 ComplexRendererFns selectShiftB_32(
const MachineOperand &Root)
const;
374 ComplexRendererFns selectShiftA_64(
const MachineOperand &Root)
const;
375 ComplexRendererFns selectShiftB_64(
const MachineOperand &Root)
const;
377 template <
unsigned ShiftW
idth>
379 ComplexRendererFns select12BitValueWithLeftShift(
uint64_t Immed)
const;
381 ComplexRendererFns selectNegArithImmed(
MachineOperand &Root)
const;
384 unsigned Size)
const;
386 ComplexRendererFns selectAddrModeUnscaled8(
MachineOperand &Root)
const {
387 return selectAddrModeUnscaled(Root, 1);
389 ComplexRendererFns selectAddrModeUnscaled16(
MachineOperand &Root)
const {
390 return selectAddrModeUnscaled(Root, 2);
392 ComplexRendererFns selectAddrModeUnscaled32(
MachineOperand &Root)
const {
393 return selectAddrModeUnscaled(Root, 4);
395 ComplexRendererFns selectAddrModeUnscaled64(
MachineOperand &Root)
const {
396 return selectAddrModeUnscaled(Root, 8);
398 ComplexRendererFns selectAddrModeUnscaled128(
MachineOperand &Root)
const {
399 return selectAddrModeUnscaled(Root, 16);
404 ComplexRendererFns tryFoldAddLowIntoImm(
MachineInstr &RootDef,
unsigned Size,
408 unsigned Size)
const;
410 ComplexRendererFns selectAddrModeIndexed(
MachineOperand &Root)
const {
411 return selectAddrModeIndexed(Root, Width / 8);
420 bool IsAddrOperand)
const;
423 unsigned SizeInBytes)
const;
431 bool WantsExt)
const;
432 ComplexRendererFns selectAddrModeRegisterOffset(
MachineOperand &Root)
const;
434 unsigned SizeInBytes)
const;
436 ComplexRendererFns selectAddrModeXRO(
MachineOperand &Root)
const {
437 return selectAddrModeXRO(Root, Width / 8);
441 unsigned SizeInBytes)
const;
443 ComplexRendererFns selectAddrModeWRO(
MachineOperand &Root)
const {
444 return selectAddrModeWRO(Root, Width / 8);
448 bool AllowROR =
false)
const;
450 ComplexRendererFns selectArithShiftedRegister(
MachineOperand &Root)
const {
451 return selectShiftedRegister(Root);
454 ComplexRendererFns selectLogicalShiftedRegister(
MachineOperand &Root)
const {
455 return selectShiftedRegister(Root,
true);
465 bool IsLoadStore =
false)
const;
476 ComplexRendererFns selectArithExtendedRegister(
MachineOperand &Root)
const;
479 template <
unsigned W
idth>
480 ComplexRendererFns selectCVTFixedPoint(
MachineOperand &Root)
const;
481 template <
unsigned W
idth>
482 ComplexRendererFns selectCVTFixedPosRecipOperand(
MachineOperand &Root)
const;
483 ComplexRendererFns selectCVTFixedPointBase(
const MachineOperand &Root,
485 bool isReciprocal =
false)
const;
486 ComplexRendererFns selectCVTFixedPointVec(
MachineOperand &Root)
const;
491 unsigned getFixedPointWidthFromOperand(
const MachineOperand &Root)
const;
493 int OpIdx = -1)
const;
497 unsigned Width,
bool isReciprocal)
const;
499 int OpIdx = -1)
const;
501 int OpIdx = -1)
const;
503 int OpIdx = -1)
const;
507 int OpIdx = -1)
const;
509 int OpIdx = -1)
const;
511 int OpIdx = -1)
const;
517 bool tryOptSelect(
GSelect &Sel);
524 bool isLoadStoreOfNumBytes(
const MachineInstr &
MI,
unsigned NumBytes)
const;
537 bool ProduceNonFlagSettingCondBr =
false;
546#define GET_GLOBALISEL_PREDICATES_DECL
547#include "AArch64GenGlobalISel.inc"
548#undef GET_GLOBALISEL_PREDICATES_DECL
552#define GET_GLOBALISEL_TEMPORARIES_DECL
553#include "AArch64GenGlobalISel.inc"
554#undef GET_GLOBALISEL_TEMPORARIES_DECL
559#define GET_GLOBALISEL_IMPL
560#include "AArch64GenGlobalISel.inc"
561#undef GET_GLOBALISEL_IMPL
563AArch64InstructionSelector::AArch64InstructionSelector(
566 : TM(TM), STI(STI),
TII(*STI.getInstrInfo()),
TRI(*STI.getRegisterInfo()),
569#include
"AArch64GenGlobalISel.inc"
572#include
"AArch64GenGlobalISel.inc"
584 bool GetAllRegSet =
false) {
585 if (RB.
getID() == AArch64::GPRRegBankID) {
586 if (Ty.getSizeInBits() <= 32)
587 return GetAllRegSet ? &AArch64::GPR32allRegClass
588 : &AArch64::GPR32RegClass;
589 if (Ty.getSizeInBits() == 64)
590 return GetAllRegSet ? &AArch64::GPR64allRegClass
591 : &AArch64::GPR64RegClass;
592 if (Ty.getSizeInBits() == 128)
593 return &AArch64::XSeqPairsClassRegClass;
597 if (RB.
getID() == AArch64::FPRRegBankID) {
598 switch (Ty.getSizeInBits()) {
600 return &AArch64::FPR8RegClass;
602 return &AArch64::FPR16RegClass;
604 return &AArch64::FPR32RegClass;
606 return &AArch64::FPR64RegClass;
608 return &AArch64::FPR128RegClass;
620 bool GetAllRegSet =
false) {
623 "Expected FPR regbank for scalable type size");
624 return &AArch64::ZPRRegClass;
627 unsigned RegBankID = RB.
getID();
629 if (RegBankID == AArch64::GPRRegBankID) {
631 if (SizeInBits <= 32)
632 return GetAllRegSet ? &AArch64::GPR32allRegClass
633 : &AArch64::GPR32RegClass;
634 if (SizeInBits == 64)
635 return GetAllRegSet ? &AArch64::GPR64allRegClass
636 : &AArch64::GPR64RegClass;
637 if (SizeInBits == 128)
638 return &AArch64::XSeqPairsClassRegClass;
641 if (RegBankID == AArch64::FPRRegBankID) {
644 "Unexpected scalable register size");
645 return &AArch64::ZPRRegClass;
648 switch (SizeInBits) {
652 return &AArch64::FPR8RegClass;
654 return &AArch64::FPR16RegClass;
656 return &AArch64::FPR32RegClass;
658 return &AArch64::FPR64RegClass;
660 return &AArch64::FPR128RegClass;
670 switch (
TRI.getRegSizeInBits(*RC)) {
672 SubReg = AArch64::bsub;
675 SubReg = AArch64::hsub;
678 if (RC != &AArch64::FPR32RegClass)
679 SubReg = AArch64::sub_32;
681 SubReg = AArch64::ssub;
684 SubReg = AArch64::dsub;
688 dbgs() <<
"Couldn't find appropriate subregister for register class.");
697 switch (RB.
getID()) {
698 case AArch64::GPRRegBankID:
700 case AArch64::FPRRegBankID:
723 const unsigned RegClassIDs[],
725 unsigned NumRegs = Regs.
size();
728 assert(NumRegs >= 2 && NumRegs <= 4 &&
729 "Only support between two and 4 registers in a tuple!");
731 auto *DesiredClass =
TRI->getRegClass(RegClassIDs[NumRegs - 2]);
733 MIB.
buildInstr(TargetOpcode::REG_SEQUENCE, {DesiredClass}, {});
734 for (
unsigned I = 0,
E = Regs.
size();
I <
E; ++
I) {
735 RegSequence.addUse(Regs[
I]);
736 RegSequence.addImm(SubRegs[
I]);
738 return RegSequence.getReg(0);
743 static const unsigned RegClassIDs[] = {
744 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
745 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
746 AArch64::dsub2, AArch64::dsub3};
747 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
752 static const unsigned RegClassIDs[] = {
753 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
754 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
755 AArch64::qsub2, AArch64::qsub3};
756 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
761 auto &
MBB = *
MI.getParent();
762 auto &MF = *
MBB.getParent();
763 auto &
MRI = MF.getRegInfo();
769 else if (Root.
isReg()) {
774 Immed = ValAndVReg->Value.getSExtValue();
785 if (RegBankID == AArch64::GPRRegBankID) {
787 switch (GenericOpc) {
788 case TargetOpcode::G_SHL:
789 return AArch64::LSLVWr;
790 case TargetOpcode::G_LSHR:
791 return AArch64::LSRVWr;
792 case TargetOpcode::G_ASHR:
793 return AArch64::ASRVWr;
797 }
else if (OpSize == 64) {
798 switch (GenericOpc) {
799 case TargetOpcode::G_SHL:
800 return AArch64::LSLVXr;
801 case TargetOpcode::G_LSHR:
802 return AArch64::LSRVXr;
803 case TargetOpcode::G_ASHR:
804 return AArch64::ASRVXr;
820 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
822 case AArch64::GPRRegBankID:
825 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
827 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
829 return isStore ? AArch64::STRWui : AArch64::LDRWui;
831 return isStore ? AArch64::STRXui : AArch64::LDRXui;
834 case AArch64::FPRRegBankID:
837 return isStore ? AArch64::STRBui : AArch64::LDRBui;
839 return isStore ? AArch64::STRHui : AArch64::LDRHui;
841 return isStore ? AArch64::STRSui : AArch64::LDRSui;
843 return isStore ? AArch64::STRDui : AArch64::LDRDui;
845 return isStore ? AArch64::STRQui : AArch64::LDRQui;
859 assert(SrcReg.
isValid() &&
"Expected a valid source register?");
860 assert(To &&
"Destination register class cannot be null");
861 assert(SubReg &&
"Expected a valid subregister");
865 MIB.
buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, {}, SubReg);
867 RegOp.
setReg(SubRegCopy.getReg(0));
871 if (!
I.getOperand(0).getReg().isPhysical())
880 Register DstReg =
I.getOperand(0).getReg();
881 Register SrcReg =
I.getOperand(1).getReg();
908 <<
TRI.getRegSizeInBits(DstReg,
MRI) <<
'\n');
912 if (
I.getOpcode() == TargetOpcode::G_BITCAST &&
914 if (DstRegBank.
getID() == AArch64::FPRRegBankID &&
915 SrcRegBank.
getID() == AArch64::GPRRegBankID) {
923 Register FPR32 =
MRI.createVirtualRegister(&AArch64::FPR32RegClass);
924 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::FMOVWSr))
927 I.setDesc(
TII.get(TargetOpcode::COPY));
928 I.getOperand(1).setReg(FPR32);
929 I.getOperand(1).setSubReg(AArch64::hsub);
933 if (DstRegBank.
getID() == AArch64::GPRRegBankID &&
934 SrcRegBank.
getID() == AArch64::FPRRegBankID) {
942 Register FPR32 =
MRI.createVirtualRegister(&AArch64::FPR32RegClass);
944 TII.get(TargetOpcode::SUBREG_TO_REG))
948 I.setDesc(
TII.get(AArch64::FMOVSWr));
949 I.getOperand(1).setReg(FPR32);
958 LLVM_DEBUG(
dbgs() <<
"Couldn't determine source register class\n");
962 const TypeSize SrcSize =
TRI.getRegSizeInBits(*SrcRC);
963 const TypeSize DstSize =
TRI.getRegSizeInBits(*DstRC);
964 unsigned SrcSubReg =
I.getOperand(1).getSubReg();
978 auto Copy = MIB.
buildCopy({DstTempRC}, {SrcReg});
980 }
else if (SrcSize > DstSize) {
987 }
else if (DstSize > SrcSize) {
994 Register PromoteReg =
MRI.createVirtualRegister(PromotionRC);
996 TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
1000 RegOp.
setReg(PromoteReg);
1019 if (
I.getOpcode() == TargetOpcode::G_ZEXT) {
1020 I.setDesc(
TII.get(AArch64::COPY));
1021 assert(SrcRegBank.
getID() == AArch64::GPRRegBankID);
1025 I.setDesc(
TII.get(AArch64::COPY));
1033 MachineRegisterInfo &
MRI = *MIB.
getMRI();
1036 "Expected both select operands to have the same regbank?");
1042 "Expected 32 bit or 64 bit select only?");
1043 const bool Is32Bit =
Size == 32;
1045 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1052 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1054 auto TryFoldBinOpIntoSelect = [&
Opc, Is32Bit, &
CC, &
MRI,
1069 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1086 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1105 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1127 if (!TrueCst && !FalseCst)
1130 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1131 if (TrueCst && FalseCst) {
1132 int64_t
T = TrueCst->Value.getSExtValue();
1133 int64_t
F = FalseCst->Value.getSExtValue();
1135 if (
T == 0 &&
F == 1) {
1137 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1143 if (
T == 0 &&
F == -1) {
1145 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1153 int64_t
T = TrueCst->Value.getSExtValue();
1156 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1165 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1174 int64_t
F = FalseCst->Value.getSExtValue();
1177 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1184 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1197 return &*SelectInst;
1202 MachineRegisterInfo *
MRI =
nullptr) {
1215 if (ValAndVReg && ValAndVReg->Value == 0)
1222 if (ValAndVReg && ValAndVReg->Value == 0)
1326 assert(
Reg.isValid() &&
"Expected valid register!");
1327 bool HasZext =
false;
1329 unsigned Opc =
MI->getOpcode();
1331 if (!
MI->getOperand(0).isReg() ||
1332 !
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
1340 if (
Opc == TargetOpcode::G_ANYEXT ||
Opc == TargetOpcode::G_ZEXT ||
1341 Opc == TargetOpcode::G_TRUNC) {
1342 if (
Opc == TargetOpcode::G_ZEXT)
1345 Register NextReg =
MI->getOperand(1).getReg();
1347 if (!NextReg.
isValid() || !
MRI.hasOneNonDBGUse(NextReg))
1349 TypeSize InSize =
MRI.getType(NextReg).getSizeInBits();
1359 std::optional<uint64_t>
C;
1364 case TargetOpcode::G_AND:
1365 case TargetOpcode::G_XOR: {
1366 TestReg =
MI->getOperand(1).getReg();
1367 Register ConstantReg =
MI->getOperand(2).getReg();
1378 C = VRegAndVal->Value.getZExtValue();
1380 C = VRegAndVal->Value.getSExtValue();
1384 case TargetOpcode::G_ASHR:
1385 case TargetOpcode::G_LSHR:
1386 case TargetOpcode::G_SHL: {
1387 TestReg =
MI->getOperand(1).getReg();
1391 C = VRegAndVal->Value.getSExtValue();
1403 unsigned TestRegSize =
MRI.getType(TestReg).getSizeInBits();
1407 case TargetOpcode::G_AND:
1409 if ((*
C >> Bit) & 1)
1412 case TargetOpcode::G_SHL:
1415 if (*
C <= Bit && (Bit - *
C) < TestRegSize) {
1420 case TargetOpcode::G_ASHR:
1425 if (Bit >= TestRegSize)
1426 Bit = TestRegSize - 1;
1428 case TargetOpcode::G_LSHR:
1430 if ((Bit + *
C) < TestRegSize) {
1435 case TargetOpcode::G_XOR:
1444 if ((*
C >> Bit) & 1)
1459MachineInstr *AArch64InstructionSelector::emitTestBit(
1460 Register TestReg,
uint64_t Bit,
bool IsNegative, MachineBasicBlock *DstMBB,
1461 MachineIRBuilder &MIB)
const {
1463 assert(ProduceNonFlagSettingCondBr &&
1464 "Cannot emit TB(N)Z with speculation tracking!");
1465 MachineRegisterInfo &
MRI = *MIB.
getMRI();
1472 assert(Bit < 64 &&
"Bit is too large!");
1476 bool UseWReg =
Bit < 32;
1477 unsigned NecessarySize = UseWReg ? 32 : 64;
1478 if (
Size != NecessarySize)
1479 TestReg = moveScalarRegClass(
1480 TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1483 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1484 {AArch64::TBZW, AArch64::TBNZW}};
1485 unsigned Opc = OpcTable[UseWReg][IsNegative];
1492bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1493 MachineInstr &AndInst,
bool Invert, MachineBasicBlock *DstMBB,
1494 MachineIRBuilder &MIB)
const {
1495 assert(AndInst.
getOpcode() == TargetOpcode::G_AND &&
"Expected G_AND only?");
1522 int32_t
Bit = MaybeBit->Value.exactLogBase2();
1529 emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
1533MachineInstr *AArch64InstructionSelector::emitCBZ(
Register CompareReg,
1535 MachineBasicBlock *DestMBB,
1536 MachineIRBuilder &MIB)
const {
1537 assert(ProduceNonFlagSettingCondBr &&
"CBZ does not set flags!");
1538 MachineRegisterInfo &
MRI = *MIB.
getMRI();
1540 AArch64::GPRRegBankID &&
1541 "Expected GPRs only?");
1545 assert(Width <= 64 &&
"Expected width to be at most 64?");
1546 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1547 {AArch64::CBNZW, AArch64::CBNZX}};
1548 unsigned Opc = OpcTable[IsNegative][Width == 64];
1549 auto BranchMI = MIB.
buildInstr(
Opc, {}, {CompareReg}).addMBB(DestMBB);
1554bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1555 MachineInstr &
I, MachineInstr &FCmp, MachineIRBuilder &MIB)
const {
1557 assert(
I.getOpcode() == TargetOpcode::G_BRCOND);
1565 MachineBasicBlock *DestMBB =
I.getOperand(1).getMBB();
1569 I.eraseFromParent();
1573bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1574 MachineInstr &
I, MachineInstr &ICmp, MachineIRBuilder &MIB)
const {
1576 assert(
I.getOpcode() == TargetOpcode::G_BRCOND);
1582 if (!ProduceNonFlagSettingCondBr)
1585 MachineRegisterInfo &
MRI = *MIB.
getMRI();
1586 MachineBasicBlock *DestMBB =
I.getOperand(1).getMBB();
1601 if (VRegAndVal && !AndInst) {
1602 int64_t
C = VRegAndVal->Value.getSExtValue();
1608 emitTestBit(
LHS, Bit,
false, DestMBB, MIB);
1609 I.eraseFromParent();
1617 emitTestBit(
LHS, Bit,
true, DestMBB, MIB);
1618 I.eraseFromParent();
1626 emitTestBit(
LHS, Bit,
false, DestMBB, MIB);
1627 I.eraseFromParent();
1641 if (VRegAndVal && VRegAndVal->Value == 0) {
1649 tryOptAndIntoCompareBranch(
1651 I.eraseFromParent();
1657 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1659 I.eraseFromParent();
1668bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1669 MachineInstr &
I, MachineInstr &ICmp, MachineIRBuilder &MIB)
const {
1671 assert(
I.getOpcode() == TargetOpcode::G_BRCOND);
1672 if (tryOptCompareBranchFedByICmp(
I, ICmp, MIB))
1676 MachineBasicBlock *DestMBB =
I.getOperand(1).getMBB();
1683 I.eraseFromParent();
1687bool AArch64InstructionSelector::selectCompareBranch(
1689 Register CondReg =
I.getOperand(0).getReg();
1694 if (CCMIOpc == TargetOpcode::G_FCMP)
1695 return selectCompareBranchFedByFCmp(
I, *CCMI, MIB);
1696 if (CCMIOpc == TargetOpcode::G_ICMP)
1697 return selectCompareBranchFedByICmp(
I, *CCMI, MIB);
1702 if (ProduceNonFlagSettingCondBr) {
1703 emitTestBit(CondReg, 0,
true,
1704 I.getOperand(1).getMBB(), MIB);
1705 I.eraseFromParent();
1715 .
addMBB(
I.getOperand(1).getMBB());
1716 I.eraseFromParent();
1725 assert(
MRI.getType(
Reg).isVector() &&
"Expected a *vector* shift operand");
1729 return std::nullopt;
1739 return std::nullopt;
1742 if (
Imm >= SrcTy.getElementType().getSizeInBits())
1743 return std::nullopt;
1747bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &
I,
1748 MachineRegisterInfo &
MRI) {
1749 assert(
I.getOpcode() == TargetOpcode::G_SHL);
1750 Register DstReg =
I.getOperand(0).getReg();
1752 Register Src1Reg =
I.getOperand(1).getReg();
1753 Register Src2Reg =
I.getOperand(2).getReg();
1764 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1766 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1768 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1770 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1772 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1774 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1776 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1788 I.eraseFromParent();
1792bool AArch64InstructionSelector::selectVectorAshrLshr(
1793 MachineInstr &
I, MachineRegisterInfo &
MRI) {
1794 assert(
I.getOpcode() == TargetOpcode::G_ASHR ||
1795 I.getOpcode() == TargetOpcode::G_LSHR);
1796 Register DstReg =
I.getOperand(0).getReg();
1798 Register Src1Reg =
I.getOperand(1).getReg();
1799 Register Src2Reg =
I.getOperand(2).getReg();
1804 bool IsASHR =
I.getOpcode() == TargetOpcode::G_ASHR;
1814 unsigned NegOpc = 0;
1816 getRegClassForTypeOnBank(Ty, RBI.
getRegBank(AArch64::FPRRegBankID));
1818 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1819 NegOpc = AArch64::NEGv2i64;
1821 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1822 NegOpc = AArch64::NEGv4i32;
1824 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1825 NegOpc = AArch64::NEGv2i32;
1827 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1828 NegOpc = AArch64::NEGv4i16;
1830 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1831 NegOpc = AArch64::NEGv8i16;
1833 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1834 NegOpc = AArch64::NEGv16i8;
1836 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1837 NegOpc = AArch64::NEGv8i8;
1843 auto Neg = MIB.
buildInstr(NegOpc, {RC}, {Src2Reg});
1847 I.eraseFromParent();
1851bool AArch64InstructionSelector::selectVaStartAAPCS(
1861 const AArch64FunctionInfo *FuncInfo = MF.
getInfo<AArch64FunctionInfo>();
1863 const auto *PtrRegClass =
1864 STI.
isTargetILP32() ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
1866 const MCInstrDesc &MCIDAddAddr =
1868 const MCInstrDesc &MCIDStoreAddr =
1880 const auto VAList =
I.getOperand(0).getReg();
1883 unsigned OffsetBytes = 0;
1887 const auto PushAddress = [&](
const int FrameIndex,
const int64_t
Imm) {
1889 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(), MCIDAddAddr)
1896 const auto *MMO = *
I.memoperands_begin();
1897 MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(), MCIDStoreAddr)
1900 .
addImm(OffsetBytes / PtrSize)
1902 MMO->getPointerInfo().getWithOffset(OffsetBytes),
1906 OffsetBytes += PtrSize;
1922 const auto PushIntConstant = [&](
const int32_t
Value) {
1923 constexpr int IntSize = 4;
1926 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::MOVi32imm))
1931 const auto *MMO = *
I.memoperands_begin();
1932 MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::STRWui))
1935 .
addImm(OffsetBytes / IntSize)
1937 MMO->getPointerInfo().getWithOffset(OffsetBytes),
1940 OffsetBytes += IntSize;
1944 PushIntConstant(-
static_cast<int32_t
>(GPRSize));
1947 PushIntConstant(-
static_cast<int32_t
>(FPRSize));
1951 I.eraseFromParent();
1955bool AArch64InstructionSelector::selectVaStartDarwin(
1957 AArch64FunctionInfo *FuncInfo = MF.
getInfo<AArch64FunctionInfo>();
1958 Register ListReg =
I.getOperand(0).getReg();
1963 if (MF.
getSubtarget<AArch64Subtarget>().isCallingConvWin64(
1971 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::ADDXri))
1979 MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::STRXui))
1986 I.eraseFromParent();
1990void AArch64InstructionSelector::materializeLargeCMVal(
1991 MachineInstr &
I,
const Value *
V,
unsigned OpFlags) {
1996 auto MovZ = MIB.
buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
2011 GV, MovZ->getOperand(1).getOffset(), Flags));
2015 MovZ->getOperand(1).getOffset(), Flags));
2021 Register DstReg = BuildMovK(MovZ.getReg(0),
2027bool AArch64InstructionSelector::preISelLower(MachineInstr &
I) {
2032 switch (
I.getOpcode()) {
2033 case TargetOpcode::G_CONSTANT: {
2034 Register DefReg =
I.getOperand(0).getReg();
2041 APInt Val =
I.getOperand(1).getCImm()->getValue().zext(32);
2042 I.getOperand(1).setCImm(
2047 I.getOperand(0).setReg(WideReg);
2056 if (PtrSize != 32 && PtrSize != 64)
2062 case TargetOpcode::G_STORE: {
2063 bool Changed = contractCrossBankCopyIntoStore(
I,
MRI);
2064 MachineOperand &SrcOp =
I.getOperand(0);
2077 case TargetOpcode::G_PTR_ADD: {
2081 if (TL->shouldPreservePtrArith(MF.
getFunction(), EVT()))
2083 return convertPtrAddToAdd(
I,
MRI);
2085 case TargetOpcode::G_LOAD: {
2090 Register DstReg =
I.getOperand(0).getReg();
2097 case TargetOpcode::G_VECREDUCE_ADD:
2098 case TargetOpcode::G_VECREDUCE_SMAX:
2099 case TargetOpcode::G_VECREDUCE_SMIN:
2100 case TargetOpcode::G_VECREDUCE_UMAX:
2101 case TargetOpcode::G_VECREDUCE_UMIN: {
2104 Register DstReg =
I.getOperand(0).getReg();
2106 if (DstRB.
getID() != AArch64::GPRRegBankID)
2111 getRegClassForTypeOnBank(DstTy, DstRB,
true);
2117 I.getOperand(0).setReg(FPRDst);
2119 BuildMI(
MBB, std::next(
I.getIterator()), MIMetadata(
I),
2120 TII.get(TargetOpcode::COPY), DstReg)
2124 case TargetOpcode::G_SPLAT_VECTOR: {
2126 LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
2133 I.getOperand(1).setReg(NewSrc.getReg(0));
2136 case AArch64::G_INSERT_VECTOR_ELT: {
2137 LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
2138 LLT SrcVecTy =
MRI.
getType(
I.getOperand(1).getReg());
2147 I.getOperand(2).setReg(NewSrc.getReg(0));
2151 Register EltReg =
I.getOperand(2).getReg();
2159 I.getOperand(2).setReg(NewElt.getReg(0));
2164 case TargetOpcode::G_UITOFP:
2165 case TargetOpcode::G_SITOFP: {
2170 Register SrcReg =
I.getOperand(1).getReg();
2172 LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
2181 I.getOperand(1).setReg(
Copy.getReg(0));
2183 getRegClassForTypeOnBank(
2184 SrcTy, RBI.
getRegBank(AArch64::FPRRegBankID)));
2186 if (
I.getOpcode() == TargetOpcode::G_SITOFP)
2187 I.setDesc(
TII.get(AArch64::G_SITOF));
2189 I.setDesc(
TII.get(AArch64::G_UITOF));
2207bool AArch64InstructionSelector::convertPtrAddToAdd(
2208 MachineInstr &
I, MachineRegisterInfo &
MRI) {
2209 assert(
I.getOpcode() == TargetOpcode::G_PTR_ADD &&
"Expected G_PTR_ADD");
2210 Register DstReg =
I.getOperand(0).getReg();
2211 Register AddOp1Reg =
I.getOperand(1).getReg();
2216 const LLT CastPtrTy = PtrTy.
isVector()
2228 I.setDesc(
TII.get(TargetOpcode::G_ADD));
2230 I.getOperand(1).setReg(PtrToInt.getReg(0));
2231 if (!select(*PtrToInt)) {
2232 LLVM_DEBUG(
dbgs() <<
"Failed to select G_PTRTOINT in convertPtrAddToAdd");
2241 I.getOperand(2).setReg(NegatedReg);
2242 I.setDesc(
TII.get(TargetOpcode::G_SUB));
2246bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &
I,
2247 MachineRegisterInfo &
MRI) {
2251 assert(
I.getOpcode() == TargetOpcode::G_SHL &&
"unexpected op");
2252 const auto &MO =
I.getOperand(2);
2257 const LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
2261 auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
2262 auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);
2264 if (!Imm1Fn || !Imm2Fn)
2268 MIB.
buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2269 {
I.getOperand(0).getReg()}, {
I.getOperand(1).getReg()});
2271 for (
auto &RenderFn : *Imm1Fn)
2273 for (
auto &RenderFn : *Imm2Fn)
2276 I.eraseFromParent();
2281bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2282 MachineInstr &
I, MachineRegisterInfo &
MRI) {
2283 assert(
I.getOpcode() == TargetOpcode::G_STORE &&
"Expected G_STORE");
2302 Register StoreSrcReg =
I.getOperand(0).getReg();
2319 I.getOperand(0).setReg(DefDstReg);
2323bool AArch64InstructionSelector::earlySelect(MachineInstr &
I) {
2324 assert(
I.getParent() &&
"Instruction should be in a basic block!");
2325 assert(
I.getParent()->getParent() &&
"Instruction should be in a function!");
2331 switch (
I.getOpcode()) {
2332 case TargetOpcode::G_SPLAT_VECTOR: {
2333 Register Dst =
I.getOperand(0).getReg();
2339 Register Src =
I.getOperand(1).getReg();
2341 Src,
MRI,
true,
true);
2352 if (!emitConstantVector(Dst, CV,
Imm,
MRI))
2354 I.eraseFromParent();
2357 case TargetOpcode::G_SEXT:
2360 if (selectUSMovFromExtend(
I,
MRI))
2363 case TargetOpcode::G_BR:
2365 case TargetOpcode::G_SHL:
2366 return earlySelectSHL(
I,
MRI);
2367 case TargetOpcode::G_CONSTANT: {
2368 bool IsZero =
false;
2369 if (
I.getOperand(1).isCImm())
2370 IsZero =
I.getOperand(1).getCImm()->isZero();
2371 else if (
I.getOperand(1).isImm())
2372 IsZero =
I.getOperand(1).getImm() == 0;
2377 Register DefReg =
I.getOperand(0).getReg();
2380 I.getOperand(1).ChangeToRegister(AArch64::XZR,
false);
2383 I.getOperand(1).ChangeToRegister(AArch64::WZR,
false);
2388 I.setDesc(
TII.get(TargetOpcode::COPY));
2392 case TargetOpcode::G_ADD: {
2401 Register AddDst =
I.getOperand(0).getReg();
2402 Register AddLHS =
I.getOperand(1).getReg();
2403 Register AddRHS =
I.getOperand(2).getReg();
2413 auto MatchCmp = [&](
Register Reg) -> MachineInstr * {
2433 MachineInstr *
Cmp = MatchCmp(AddRHS);
2437 Cmp = MatchCmp(AddRHS);
2441 auto &PredOp =
Cmp->getOperand(1);
2443 emitIntegerCompare(
Cmp->getOperand(2),
2444 Cmp->getOperand(3), PredOp, MIB);
2448 emitCSINC(AddDst, AddLHS, AddLHS, InvCC, MIB);
2449 I.eraseFromParent();
2452 case TargetOpcode::G_OR: {
2456 Register Dst =
I.getOperand(0).getReg();
2475 if (ShiftImm >
Size || ((1ULL << ShiftImm) - 1ULL) !=
uint64_t(MaskImm))
2478 int64_t Immr =
Size - ShiftImm;
2479 int64_t Imms =
Size - ShiftImm - 1;
2480 unsigned Opc =
Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2481 emitInstr(
Opc, {Dst}, {MaskSrc, ShiftSrc, Immr, Imms}, MIB);
2482 I.eraseFromParent();
2485 case TargetOpcode::G_FENCE: {
2486 if (
I.getOperand(1).getImm() == 0)
2490 .
addImm(
I.getOperand(0).getImm() == 4 ? 0x9 : 0xb);
2491 I.eraseFromParent();
2494 case TargetOpcode::G_BRINDIRECT: {
2496 if (std::optional<uint16_t> BADisc =
2498 auto MI = MIB.
buildInstr(AArch64::BRA, {}, {
I.getOperand(0).getReg()});
2502 I.eraseFromParent();
2515bool AArch64InstructionSelector::select(MachineInstr &
I) {
2516 assert(
I.getParent() &&
"Instruction should be in a basic block!");
2517 assert(
I.getParent()->getParent() &&
"Instruction should be in a function!");
2526 LLVM_DEBUG(
dbgs() <<
"AArch64 GISel does not support strict-align yet\n");
2532 unsigned Opcode =
I.getOpcode();
2534 if (!
I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2537 if (Opcode == TargetOpcode::LOAD_STACK_GUARD) {
2542 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2543 const Register DefReg =
I.getOperand(0).getReg();
2557 DefRC = getRegClassForTypeOnBank(DefTy, RB);
2564 I.setDesc(
TII.get(TargetOpcode::PHI));
2576 if (
I.getNumOperands() !=
I.getNumExplicitOperands()) {
2578 dbgs() <<
"Generic instruction has unexpected implicit operands\n");
2585 if (preISelLower(
I)) {
2586 Opcode =
I.getOpcode();
2597 if (selectImpl(
I, *CoverageInfo))
2601 I.getOperand(0).isReg() ?
MRI.
getType(
I.getOperand(0).getReg()) : LLT{};
2604 case TargetOpcode::G_SBFX:
2605 case TargetOpcode::G_UBFX: {
2606 static const unsigned OpcTable[2][2] = {
2607 {AArch64::UBFMWri, AArch64::UBFMXri},
2608 {AArch64::SBFMWri, AArch64::SBFMXri}};
2609 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2611 unsigned Opc = OpcTable[IsSigned][
Size == 64];
2614 assert(Cst1 &&
"Should have gotten a constant for src 1?");
2617 assert(Cst2 &&
"Should have gotten a constant for src 2?");
2618 auto LSB = Cst1->Value.getZExtValue();
2619 auto Width = Cst2->Value.getZExtValue();
2623 .
addImm(LSB + Width - 1);
2624 I.eraseFromParent();
2628 case TargetOpcode::G_BRCOND:
2629 return selectCompareBranch(
I, MF,
MRI);
2631 case TargetOpcode::G_BRJT:
2632 return selectBrJT(
I,
MRI);
2634 case AArch64::G_ADD_LOW: {
2640 if (BaseMI->
getOpcode() != AArch64::ADRP) {
2641 I.setDesc(
TII.get(AArch64::ADDXri));
2647 "Expected small code model");
2649 auto Op2 =
I.getOperand(2);
2650 auto MovAddr = MIB.
buildInstr(AArch64::MOVaddr, {
I.getOperand(0)}, {})
2651 .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
2652 Op1.getTargetFlags())
2654 Op2.getTargetFlags());
2655 I.eraseFromParent();
2660 case TargetOpcode::G_FCONSTANT: {
2661 const Register DefReg =
I.getOperand(0).getReg();
2668 (DefSize == 16 || DefSize == 32 || DefSize == 64 || DefSize == 128) &&
2669 "Unexpected destination size for G_FCONSTANT?");
2672 const ConstantFP *FPImm =
I.getOperand(1).getFPImm();
2673 APInt
Imm = DefSize >= 64
2676 if (
auto *Mov = emitConstantMOVVector(
2677 DefSize >= 64 ? DstOp(DefReg) : DstOp(&AArch64::FPR64RegClass),
Imm,
2680 MIB.
buildInstr(TargetOpcode::COPY, {DefReg}, {})
2681 .addReg(Mov->getOperand(0).getReg(), {},
2682 DefSize == 16 ? AArch64::hsub : AArch64::ssub);
2685 I.eraseFromParent();
2691 bool OptForSize = shouldOptForSize(&MF);
2693 if ((DefSize == 32 || DefSize == 64) &&
2698 DefSize == 64 ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass);
2699 MachineOperand &RegOp =
I.getOperand(0);
2705 LLVM_DEBUG(
dbgs() <<
"Failed to constrain G_FCONSTANT def operand\n");
2709 MachineOperand &ImmOp =
I.getOperand(1);
2713 const unsigned MovOpc =
2714 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2715 I.setDesc(
TII.get(MovOpc));
2723 LLVM_DEBUG(
dbgs() <<
"Failed to load double constant pool entry\n");
2726 MIB.
buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
2727 I.eraseFromParent();
2730 case TargetOpcode::G_EXTRACT: {
2731 Register DstReg =
I.getOperand(0).getReg();
2732 Register SrcReg =
I.getOperand(1).getReg();
2746 unsigned Offset =
I.getOperand(2).getImm();
2755 if (SrcRB.
getID() == AArch64::GPRRegBankID) {
2757 MIB.
buildInstr(TargetOpcode::COPY, {DstReg}, {})
2759 Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2761 AArch64::GPR64RegClass, NewI->getOperand(0));
2762 I.eraseFromParent();
2768 unsigned LaneIdx =
Offset / 64;
2769 MachineInstr *Extract = emitExtractVectorElt(
2770 DstReg, DstRB,
LLT::scalar(64), SrcReg, LaneIdx, MIB);
2773 I.eraseFromParent();
2777 I.setDesc(
TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2778 MachineInstrBuilder(MF,
I).addImm(
I.getOperand(2).getImm() +
2783 "unexpected G_EXTRACT types");
2790 MIB.
buildInstr(TargetOpcode::COPY, {
I.getOperand(0).getReg()}, {})
2791 .addReg(DstReg, {}, AArch64::sub_32);
2793 AArch64::GPR32RegClass,
MRI);
2794 I.getOperand(0).setReg(DstReg);
2800 case TargetOpcode::G_INSERT: {
2801 LLT SrcTy =
MRI.
getType(
I.getOperand(2).getReg());
2802 LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
2809 I.setDesc(
TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2810 unsigned LSB =
I.getOperand(3).getImm();
2812 I.getOperand(3).setImm((DstSize - LSB) % DstSize);
2813 MachineInstrBuilder(MF,
I).addImm(Width - 1);
2817 "unexpected G_INSERT types");
2824 TII.get(AArch64::SUBREG_TO_REG))
2826 .
addUse(
I.getOperand(2).getReg())
2827 .
addImm(AArch64::sub_32);
2829 AArch64::GPR32RegClass,
MRI);
2830 I.getOperand(2).setReg(SrcReg);
2835 case TargetOpcode::G_FRAME_INDEX: {
2842 I.setDesc(
TII.get(AArch64::ADDXri));
2852 case TargetOpcode::G_GLOBAL_VALUE: {
2853 const GlobalValue *GV =
nullptr;
2855 if (
I.getOperand(1).isSymbol()) {
2856 OpFlags =
I.getOperand(1).getTargetFlags();
2862 return selectTLSGlobalValue(
I,
MRI);
2868 bool IsGOTSigned = MF.
getInfo<AArch64FunctionInfo>()->hasELFSignedGOT();
2869 I.setDesc(
TII.get(IsGOTSigned ? AArch64::LOADgotAUTH : AArch64::LOADgot));
2870 I.getOperand(1).setTargetFlags(OpFlags);
2871 I.addImplicitDefUseOperands(MF);
2872 I.setImplicitPhysRegDefsDead();
2876 materializeLargeCMVal(
I, GV, OpFlags);
2877 I.eraseFromParent();
2880 I.setDesc(
TII.get(AArch64::ADR));
2881 I.getOperand(1).setTargetFlags(OpFlags);
2883 I.setDesc(
TII.get(AArch64::MOVaddr));
2885 MachineInstrBuilder MIB(MF,
I);
2886 MIB.addGlobalAddress(GV,
I.getOperand(1).getOffset(),
2893 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2894 return selectPtrAuthGlobalValue(
I,
MRI);
2896 case TargetOpcode::G_ZEXTLOAD:
2897 case TargetOpcode::G_LOAD:
2898 case TargetOpcode::G_STORE: {
2900 bool IsZExtLoad =
I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2915 assert(MemSizeInBytes <= 8 &&
2916 "128-bit atomics should already be custom-legalized");
2919 static constexpr unsigned LDAPROpcodes[] = {
2920 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2921 static constexpr unsigned LDAROpcodes[] = {
2922 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2923 ArrayRef<unsigned> Opcodes =
2924 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2927 I.setDesc(
TII.get(Opcodes[
Log2_32(MemSizeInBytes)]));
2929 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2930 AArch64::STLRW, AArch64::STLRX};
2935 MIB.
buildInstr(TargetOpcode::COPY, {NewVal}, {})
2936 .addReg(
I.getOperand(0).getReg(), {}, AArch64::sub_32);
2937 I.getOperand(0).setReg(NewVal);
2939 I.setDesc(
TII.get(Opcodes[
Log2_32(MemSizeInBytes)]));
2950 "Load/Store pointer operand isn't a GPR");
2952 "Load/Store pointer operand isn't a pointer");
2962 RB.
getID() == AArch64::FPRRegBankID) {
2965 auto *RC = getRegClassForTypeOnBank(MemTy, RB);
2971 .addReg(ValReg, {}, SubReg)
2978 if (RB.
getID() == AArch64::FPRRegBankID) {
2981 auto *RC = getRegClassForTypeOnBank(MemTy, RB);
2991 MIB.
buildInstr(AArch64::SUBREG_TO_REG, {OldDst}, {})
2994 auto SubRegRC = getRegClassForTypeOnBank(
MRI.
getType(OldDst), RB);
3003 auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
3005 const unsigned NewOpc =
3007 if (NewOpc ==
I.getOpcode())
3011 selectAddrModeIndexed(
I.getOperand(1), MemSizeInBytes);
3014 I.setDesc(
TII.get(NewOpc));
3020 auto NewInst = MIB.
buildInstr(NewOpc, {}, {},
I.getFlags());
3021 Register CurValReg =
I.getOperand(0).getReg();
3022 IsStore ? NewInst.addUse(CurValReg) : NewInst.addDef(CurValReg);
3023 NewInst.cloneMemRefs(
I);
3024 for (
auto &Fn : *AddrModeFns)
3026 I.eraseFromParent();
3030 MachineInstr *
LoadStore = SelectLoadStoreAddressingMode();
3035 if (Opcode == TargetOpcode::G_STORE) {
3038 if (CVal && CVal->Value == 0) {
3040 case AArch64::STRWui:
3041 case AArch64::STRHHui:
3042 case AArch64::STRBBui:
3043 LoadStore->getOperand(0).setReg(AArch64::WZR);
3045 case AArch64::STRXui:
3046 LoadStore->getOperand(0).setReg(AArch64::XZR);
3052 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3053 ValTy ==
LLT::scalar(64) && MemSizeInBits == 32)) {
3065 MIB.
buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
3067 .
addImm(AArch64::sub_32);
3076 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3077 case TargetOpcode::G_INDEXED_SEXTLOAD:
3078 return selectIndexedExtLoad(
I,
MRI);
3079 case TargetOpcode::G_INDEXED_LOAD:
3080 return selectIndexedLoad(
I,
MRI);
3081 case TargetOpcode::G_INDEXED_STORE:
3084 case TargetOpcode::G_LSHR:
3085 case TargetOpcode::G_ASHR:
3087 return selectVectorAshrLshr(
I,
MRI);
3089 case TargetOpcode::G_SHL: {
3090 if (Opcode == TargetOpcode::G_SHL &&
3092 return selectVectorSHL(
I,
MRI);
3099 Register SrcReg =
I.getOperand(1).getReg();
3100 Register ShiftReg =
I.getOperand(2).getReg();
3108 .addReg(ShiftReg, {}, AArch64::sub_32);
3110 I.getOperand(2).setReg(
Trunc.getReg(0));
3115 const Register DefReg =
I.getOperand(0).getReg();
3119 if (NewOpc ==
I.getOpcode())
3122 I.setDesc(
TII.get(NewOpc));
3130 case TargetOpcode::G_PTR_ADD: {
3131 emitADD(
I.getOperand(0).getReg(),
I.getOperand(1),
I.getOperand(2), MIB);
3132 I.eraseFromParent();
3136 case TargetOpcode::G_SADDE:
3137 case TargetOpcode::G_UADDE:
3138 case TargetOpcode::G_SSUBE:
3139 case TargetOpcode::G_USUBE:
3140 case TargetOpcode::G_SADDO:
3141 case TargetOpcode::G_UADDO:
3142 case TargetOpcode::G_SSUBO:
3143 case TargetOpcode::G_USUBO:
3144 return selectOverflowOp(
I,
MRI);
3146 case TargetOpcode::G_PTRMASK: {
3147 Register MaskReg =
I.getOperand(2).getReg();
3154 I.setDesc(
TII.get(AArch64::ANDXri));
3155 I.getOperand(2).ChangeToImmediate(
3161 case TargetOpcode::G_PTRTOINT:
3162 case TargetOpcode::G_TRUNC: {
3163 const LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
3164 const LLT SrcTy =
MRI.
getType(
I.getOperand(1).getReg());
3166 const Register DstReg =
I.getOperand(0).getReg();
3167 const Register SrcReg =
I.getOperand(1).getReg();
3174 dbgs() <<
"G_TRUNC/G_PTRTOINT input/output on different banks\n");
3178 if (DstRB.
getID() == AArch64::GPRRegBankID) {
3189 LLVM_DEBUG(
dbgs() <<
"Failed to constrain G_TRUNC/G_PTRTOINT\n");
3193 if (DstRC == SrcRC) {
3195 }
else if (Opcode == TargetOpcode::G_TRUNC && DstTy ==
LLT::scalar(32) &&
3199 }
else if (DstRC == &AArch64::GPR32RegClass &&
3200 SrcRC == &AArch64::GPR64RegClass) {
3201 I.getOperand(1).setSubReg(AArch64::sub_32);
3204 dbgs() <<
"Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3208 I.setDesc(
TII.get(TargetOpcode::COPY));
3210 }
else if (DstRB.
getID() == AArch64::FPRRegBankID) {
3213 I.setDesc(
TII.get(AArch64::XTNv4i16));
3219 MachineInstr *Extract = emitExtractVectorElt(
3223 I.eraseFromParent();
3228 if (Opcode == TargetOpcode::G_PTRTOINT) {
3229 assert(DstTy.
isVector() &&
"Expected an FPR ptrtoint to be a vector");
3230 I.setDesc(
TII.get(TargetOpcode::COPY));
3238 case TargetOpcode::G_ANYEXT: {
3239 if (selectUSMovFromExtend(
I,
MRI))
3242 const Register DstReg =
I.getOperand(0).getReg();
3243 const Register SrcReg =
I.getOperand(1).getReg();
3246 if (RBDst.
getID() != AArch64::GPRRegBankID) {
3248 <<
", expected: GPR\n");
3253 if (RBSrc.
getID() != AArch64::GPRRegBankID) {
3255 <<
", expected: GPR\n");
3262 LLVM_DEBUG(
dbgs() <<
"G_ANYEXT operand has no size, not a gvreg?\n");
3266 if (DstSize != 64 && DstSize > 32) {
3268 <<
", expected: 32 or 64\n");
3278 .
addImm(AArch64::sub_32);
3279 I.getOperand(1).setReg(ExtSrc);
3284 case TargetOpcode::G_ZEXT:
3285 case TargetOpcode::G_SEXT_INREG:
3286 case TargetOpcode::G_SEXT: {
3287 if (selectUSMovFromExtend(
I,
MRI))
3290 unsigned Opcode =
I.getOpcode();
3291 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3292 const Register DefReg =
I.getOperand(0).getReg();
3293 Register SrcReg =
I.getOperand(1).getReg();
3301 if (Opcode == TargetOpcode::G_SEXT_INREG)
3302 SrcSize =
I.getOperand(2).getImm();
3308 AArch64::GPRRegBankID &&
3309 "Unexpected ext regbank");
3323 if (LoadMI && IsGPR) {
3324 const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
3325 unsigned BytesLoaded = MemOp->getSize().getValue();
3332 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3335 const Register ZReg = AArch64::WZR;
3336 MIB.
buildInstr(AArch64::ORRWrs, {SubregToRegSrc}, {ZReg, SrcReg})
3339 MIB.
buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
3340 .addUse(SubregToRegSrc)
3341 .
addImm(AArch64::sub_32);
3345 LLVM_DEBUG(
dbgs() <<
"Failed to constrain G_ZEXT destination\n");
3355 I.eraseFromParent();
3360 if (DstSize == 64) {
3361 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3369 SrcReg = MIB.
buildInstr(AArch64::SUBREG_TO_REG,
3370 {&AArch64::GPR64RegClass}, {})
3376 ExtI = MIB.
buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3380 }
else if (DstSize <= 32) {
3381 ExtI = MIB.
buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3390 I.eraseFromParent();
3394 case TargetOpcode::G_FREEZE:
3397 case TargetOpcode::G_INTTOPTR:
3402 case TargetOpcode::G_BITCAST:
3408 case TargetOpcode::G_SELECT: {
3410 const Register CondReg = Sel.getCondReg();
3412 const Register FReg = Sel.getFalseReg();
3414 if (tryOptSelect(Sel))
3420 auto TstMI = MIB.
buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
3425 Sel.eraseFromParent();
3428 case TargetOpcode::G_ICMP: {
3438 auto &PredOp =
I.getOperand(1);
3439 emitIntegerCompare(
I.getOperand(2),
I.getOperand(3), PredOp, MIB);
3443 emitCSINC(
I.getOperand(0).getReg(), AArch64::WZR,
3444 AArch64::WZR, InvCC, MIB);
3445 I.eraseFromParent();
3449 case TargetOpcode::G_FCMP: {
3452 if (!emitFPCompare(
I.getOperand(2).getReg(),
I.getOperand(3).getReg(), MIB,
3454 !emitCSetForFCmp(
I.getOperand(0).getReg(),
Pred, MIB))
3456 I.eraseFromParent();
3459 case TargetOpcode::G_VASTART:
3461 : selectVaStartAAPCS(
I, MF,
MRI);
3462 case TargetOpcode::G_INTRINSIC:
3463 return selectIntrinsic(
I,
MRI);
3464 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3465 return selectIntrinsicWithSideEffects(
I,
MRI);
3466 case TargetOpcode::G_IMPLICIT_DEF: {
3467 I.setDesc(
TII.get(TargetOpcode::IMPLICIT_DEF));
3468 const LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
3469 const Register DstReg =
I.getOperand(0).getReg();
3475 case TargetOpcode::G_BLOCK_ADDR: {
3476 Function *BAFn =
I.getOperand(1).getBlockAddress()->getFunction();
3477 if (std::optional<uint16_t> BADisc =
3487 AArch64::GPR64RegClass,
MRI);
3488 I.eraseFromParent();
3492 materializeLargeCMVal(
I,
I.getOperand(1).getBlockAddress(), 0);
3493 I.eraseFromParent();
3496 I.setDesc(
TII.get(AArch64::MOVaddrBA));
3497 auto MovMI =
BuildMI(
MBB,
I,
I.getDebugLoc(),
TII.get(AArch64::MOVaddrBA),
3498 I.getOperand(0).getReg())
3502 I.getOperand(1).getBlockAddress(), 0,
3504 I.eraseFromParent();
3509 case TargetOpcode::G_BUILD_VECTOR:
3510 return selectBuildVector(
I,
MRI);
3511 case TargetOpcode::G_MERGE_VALUES:
3513 case TargetOpcode::G_UNMERGE_VALUES:
3515 case TargetOpcode::G_SHUFFLE_VECTOR:
3516 return selectShuffleVector(
I,
MRI);
3517 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3518 return selectExtractElt(
I,
MRI);
3519 case TargetOpcode::G_CONCAT_VECTORS:
3520 return selectConcatVectors(
I,
MRI);
3521 case TargetOpcode::G_JUMP_TABLE:
3522 return selectJumpTable(
I,
MRI);
3523 case TargetOpcode::G_MEMCPY:
3524 case TargetOpcode::G_MEMCPY_INLINE:
3525 case TargetOpcode::G_MEMMOVE:
3526 case TargetOpcode::G_MEMSET:
3527 case TargetOpcode::G_MEMSET_INLINE:
3528 assert(STI.hasMOPS() &&
"Shouldn't get here without +mops feature");
3529 return selectMOPS(
I,
MRI);
3535bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &
I) {
3536 MachineIRBuilderState OldMIBState = MIB.
getState();
3542bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
3543 MachineRegisterInfo &
MRI) {
3546 case TargetOpcode::G_MEMCPY:
3547 case TargetOpcode::G_MEMCPY_INLINE:
3548 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3550 case TargetOpcode::G_MEMMOVE:
3551 Mopcode = AArch64::MOPSMemoryMovePseudo;
3553 case TargetOpcode::G_MEMSET:
3554 case TargetOpcode::G_MEMSET_INLINE:
3556 Mopcode = AArch64::MOPSMemorySetPseudo;
3569 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3570 const auto &SrcValRegClass =
3571 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3589 MIB.
buildInstr(Mopcode, {DefDstPtr, DefSize},
3590 {DstPtrCopy, SizeCopy, SrcValCopy})
3594 MIB.
buildInstr(Mopcode, {DefDstPtr, DefSrcPtr, DefSize},
3595 {DstPtrCopy, SrcValCopy, SizeCopy})
3603bool AArch64InstructionSelector::selectBrJT(MachineInstr &
I,
3604 MachineRegisterInfo &
MRI) {
3605 assert(
I.getOpcode() == TargetOpcode::G_BRJT &&
"Expected G_BRJT");
3606 Register JTAddr =
I.getOperand(0).getReg();
3607 unsigned JTI =
I.getOperand(1).getIndex();
3610 MF->
getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4,
nullptr);
3622 "jump table hardening only supported on MachO/ELF");
3630 I.eraseFromParent();
3637 auto JumpTableInst = MIB.
buildInstr(AArch64::JumpTableDest32,
3638 {TargetReg, ScratchReg}, {JTAddr,
Index})
3639 .addJumpTableIndex(JTI);
3641 MIB.
buildInstr(TargetOpcode::JUMP_TABLE_DEBUG_INFO, {},
3642 {
static_cast<int64_t
>(JTI)});
3644 MIB.
buildInstr(AArch64::BR, {}, {TargetReg});
3645 I.eraseFromParent();
3650bool AArch64InstructionSelector::selectJumpTable(MachineInstr &
I,
3651 MachineRegisterInfo &
MRI) {
3652 assert(
I.getOpcode() == TargetOpcode::G_JUMP_TABLE &&
"Expected jump table");
3653 assert(
I.getOperand(1).isJTI() &&
"Jump table op should have a JTI!");
3655 Register DstReg =
I.getOperand(0).getReg();
3656 unsigned JTI =
I.getOperand(1).getIndex();
3659 MIB.
buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
3662 I.eraseFromParent();
3667bool AArch64InstructionSelector::selectTLSLocalExecELF(
3668 const GlobalValue *GV, MachineInstr &
I, MachineRegisterInfo &
MRI) {
3669 auto ConstrainRegOps = [&](MachineInstrBuilder MIB) {
3673 ConstrainRegOps(MIB.
buildInstr(AArch64::MOVbaseTLS, {ThreadBase}, {}));
3681 MIB.
buildInstr(AArch64::ADDXri, {I.getOperand(0).getReg()},
3692 MIB.
buildInstr(AArch64::ADDXri, {Addr}, {ThreadBase})
3696 MIB.
buildInstr(AArch64::ADDXri, {I.getOperand(0).getReg()}, {Addr})
3713 ConstrainRegOps(MIB.
buildInstr(AArch64::MOVKXi, {Addr2}, {Addr})
3718 ConstrainRegOps(MIB.
buildInstr(AArch64::ADDXrr, {I.getOperand(0).getReg()},
3719 {ThreadBase, Addr2}));
3733 ConstrainRegOps(MIB.
buildInstr(AArch64::MOVKXi, {Addr2}, {Addr})
3739 ConstrainRegOps(MIB.
buildInstr(AArch64::MOVKXi, {Addr3}, {Addr2})
3744 ConstrainRegOps(MIB.
buildInstr(AArch64::ADDXrr, {I.getOperand(0).getReg()},
3745 {ThreadBase, Addr3}));
3749 I.eraseFromParent();
3755bool AArch64InstructionSelector::selectTLSGlobalValueELF(
3756 MachineInstr &
I, MachineRegisterInfo &
MRI) {
3757 const GlobalValue *GV =
I.
getOperand(1).getGlobal();
3758 auto *FuncInfo = MF->
getInfo<AArch64FunctionInfo>();
3765 return selectTLSLocalExecELF(GV,
I,
MRI);
3767 MIB.
buildInstr(AArch64::LOADgot, {TPOff}, {})
3773 SMEAttrs
Attrs = MF->
getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
3775 !
Attrs.hasStreamingCompatibleInterface() &&
3776 "unsupported SME features reached GlobalISel TLS lowering");
3779 ? AArch64::TLSDESC_AUTH_CALLSEQ
3780 : AArch64::TLSDESC_CALLSEQ;
3799 MIB.
buildInstr(AArch64::ADDXri, {TPOff}, {Add1.getReg(0)})
3800 .addGlobalAddress(GV, 0,
3809 MIB.
buildInstr(AArch64::MOVbaseTLS, {ThreadBase}, {});
3810 auto Add = MIB.
buildInstr(AArch64::ADDXrr, {
I.getOperand(0).getReg()},
3811 {ThreadBase, TPOff});
3814 I.eraseFromParent();
3818bool AArch64InstructionSelector::selectTLSGlobalValueMachO(
3819 MachineInstr &
I, MachineRegisterInfo &
MRI) {
3820 const auto &GlobalOp =
I.getOperand(1);
3821 assert(GlobalOp.getOffset() == 0 &&
3822 "Shouldn't have an offset on TLS globals!");
3824 const GlobalValue &GV = *GlobalOp.getGlobal();
3827 MIB.
buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
3830 auto Load = MIB.
buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
3831 {LoadGOT.getReg(0)})
3842 assert(Opcode == AArch64::BLR);
3843 Opcode = AArch64::BLRAAZ;
3848 .
addUse(AArch64::X0, RegState::Implicit)
3849 .
addDef(AArch64::X0, RegState::Implicit)
3855 I.eraseFromParent();
3859bool AArch64InstructionSelector::selectTLSGlobalValue(
3860 MachineInstr &
I, MachineRegisterInfo &
MRI) {
3866 return selectTLSGlobalValueELF(
I,
MRI);
3869 return selectTLSGlobalValueMachO(
I,
MRI);
3874MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3876 MachineIRBuilder &MIRBuilder)
const {
3877 auto Undef = MIRBuilder.
buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});
3879 auto BuildFn = [&](
unsigned SubregIndex) {
3883 .addImm(SubregIndex);
3891 return BuildFn(AArch64::bsub);
3893 return BuildFn(AArch64::hsub);
3895 return BuildFn(AArch64::ssub);
3897 return BuildFn(AArch64::dsub);
3904AArch64InstructionSelector::emitNarrowVector(
Register DstReg,
Register SrcReg,
3905 MachineIRBuilder &MIB,
3906 MachineRegisterInfo &
MRI)
const {
3910 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3914 unsigned SubReg = 0;
3917 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
3923 .addReg(SrcReg, {}, SubReg);
3928bool AArch64InstructionSelector::selectMergeValues(
3929 MachineInstr &
I, MachineRegisterInfo &
MRI) {
3930 assert(
I.getOpcode() == TargetOpcode::G_MERGE_VALUES &&
"unexpected opcode");
3931 const LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
3932 const LLT SrcTy =
MRI.
getType(
I.getOperand(1).getReg());
3936 if (
I.getNumOperands() != 3)
3943 Register DstReg =
I.getOperand(0).getReg();
3944 Register Src1Reg =
I.getOperand(1).getReg();
3945 Register Src2Reg =
I.getOperand(2).getReg();
3946 auto Tmp = MIB.
buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
3947 MachineInstr *InsMI = emitLaneInsert(std::nullopt, Tmp.getReg(0), Src1Reg,
3951 MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->
getOperand(0).
getReg(),
3952 Src2Reg, 1, RB, MIB);
3957 I.eraseFromParent();
3961 if (RB.
getID() != AArch64::GPRRegBankID)
3967 auto *DstRC = &AArch64::GPR64RegClass;
3969 MachineInstr &SubRegMI = *
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
3970 TII.get(TargetOpcode::SUBREG_TO_REG))
3972 .
addUse(
I.getOperand(1).getReg())
3973 .
addImm(AArch64::sub_32);
3976 MachineInstr &SubRegMI2 = *
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
3977 TII.get(TargetOpcode::SUBREG_TO_REG))
3979 .
addUse(
I.getOperand(2).getReg())
3980 .
addImm(AArch64::sub_32);
3982 *
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::BFMXri))
3983 .
addDef(
I.getOperand(0).getReg())
3991 I.eraseFromParent();
3996 const unsigned EltSize) {
4001 CopyOpc = AArch64::DUPi8;
4002 ExtractSubReg = AArch64::bsub;
4005 CopyOpc = AArch64::DUPi16;
4006 ExtractSubReg = AArch64::hsub;
4009 CopyOpc = AArch64::DUPi32;
4010 ExtractSubReg = AArch64::ssub;
4013 CopyOpc = AArch64::DUPi64;
4014 ExtractSubReg = AArch64::dsub;
4018 LLVM_DEBUG(
dbgs() <<
"Elt size '" << EltSize <<
"' unsupported.\n");
4024MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
4025 std::optional<Register> DstReg,
const RegisterBank &DstRB, LLT ScalarTy,
4026 Register VecReg,
unsigned LaneIdx, MachineIRBuilder &MIRBuilder)
const {
4027 MachineRegisterInfo &
MRI = *MIRBuilder.
getMRI();
4028 unsigned CopyOpc = 0;
4029 unsigned ExtractSubReg = 0;
4032 dbgs() <<
"Couldn't determine lane copy opcode for instruction.\n");
4037 getRegClassForTypeOnBank(ScalarTy, DstRB,
true);
4039 LLVM_DEBUG(
dbgs() <<
"Could not determine destination register class.\n");
4046 getRegClassForTypeOnBank(VecTy, VecRB,
true);
4048 LLVM_DEBUG(
dbgs() <<
"Could not determine source register class.\n");
4058 auto Copy = MIRBuilder.
buildInstr(TargetOpcode::COPY, {*DstReg}, {})
4059 .addReg(VecReg, {}, ExtractSubReg);
4068 MachineInstr *ScalarToVector = emitScalarToVector(
4069 VecTy.
getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
4070 if (!ScalarToVector)
4075 MachineInstr *LaneCopyMI =
4076 MIRBuilder.
buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
4084bool AArch64InstructionSelector::selectExtractElt(
4085 MachineInstr &
I, MachineRegisterInfo &
MRI) {
4086 assert(
I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
4087 "unexpected opcode!");
4088 Register DstReg =
I.getOperand(0).getReg();
4090 const Register SrcReg =
I.getOperand(1).getReg();
4093 "source register size too small!");
4094 assert(!NarrowTy.
isVector() &&
"cannot extract vector into vector!");
4097 MachineOperand &LaneIdxOp =
I.getOperand(2);
4098 assert(LaneIdxOp.
isReg() &&
"Lane index operand was not a register?");
4104 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
4107 if (DstRB.
getID() == AArch64::GPRRegBankID) {
4111 Opcode = AArch64::UMOVvi8;
4114 Opcode = AArch64::UMOVvi16;
4117 Opcode = AArch64::UMOVvi32;
4124 MachineInstr *ScalarToVector = emitScalarToVector(
4125 WideTy.
getSizeInBits(), &AArch64::FPR128RegClass, SrcReg, MIB);
4126 assert(ScalarToVector &&
"Didn't expect emitScalarToVector to fail!");
4130 I.setDesc(
TII.get(Opcode));
4131 I.getOperand(2).ChangeToImmediate(LaneIdx);
4136 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
4141 I.eraseFromParent();
4145bool AArch64InstructionSelector::selectSplitVectorUnmerge(
4146 MachineInstr &
I, MachineRegisterInfo &
MRI) {
4147 unsigned NumElts =
I.getNumOperands() - 1;
4148 Register SrcReg =
I.getOperand(NumElts).getReg();
4149 const LLT NarrowTy =
MRI.
getType(
I.getOperand(0).getReg());
4152 assert(NarrowTy.
isVector() &&
"Expected an unmerge into vectors");
4154 LLVM_DEBUG(
dbgs() <<
"Unexpected vector type for vec split unmerge");
4160 const RegisterBank &DstRB =
4162 for (
unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
4163 Register Dst =
I.getOperand(OpIdx).getReg();
4164 MachineInstr *Extract =
4165 emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
4169 I.eraseFromParent();
4173bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &
I,
4174 MachineRegisterInfo &
MRI) {
4175 assert(
I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
4176 "unexpected opcode");
4180 unsigned NumElts =
I.getNumOperands() - 1;
4181 Register SrcReg =
I.getOperand(NumElts).getReg();
4182 Register LoReg =
I.getOperand(0).getReg();
4183 Register HiReg =
I.getOperand(1).getReg();
4192 LoRB.
getID() == AArch64::GPRRegBankID &&
4193 HiRB.
getID() == AArch64::GPRRegBankID &&
4194 SrcRB.
getID() == AArch64::FPRRegBankID) {
4195 MachineInstr &
Lo = *
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4196 TII.get(AArch64::UMOVvi64), LoReg)
4199 MachineInstr &
Hi = *
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4200 TII.get(AArch64::UMOVvi64), HiReg)
4205 I.eraseFromParent();
4210 if (LoRB.
getID() != AArch64::FPRRegBankID ||
4211 HiRB.
getID() != AArch64::FPRRegBankID) {
4212 LLVM_DEBUG(
dbgs() <<
"Unmerging vector-to-gpr and scalar-to-scalar "
4213 "currently unsupported.\n");
4218 "source register size too small!");
4221 return selectSplitVectorUnmerge(
I,
MRI);
4225 unsigned CopyOpc = 0;
4226 unsigned ExtractSubReg = 0;
4237 unsigned NumInsertRegs = NumElts - 1;
4243 InsertRegs.
assign(NumInsertRegs, SrcReg);
4252 unsigned SubReg = 0;
4255 assert(Found &&
"expected to find last operand's subeg idx");
4256 for (
unsigned Idx = 0;
Idx < NumInsertRegs; ++
Idx) {
4258 MachineInstr &ImpDefMI =
4259 *
BuildMI(
MBB,
I,
I.getDebugLoc(),
TII.get(TargetOpcode::IMPLICIT_DEF),
4264 MachineInstr &InsMI =
4266 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
4283 Register CopyTo =
I.getOperand(0).getReg();
4284 auto FirstCopy = MIB.
buildInstr(TargetOpcode::COPY, {CopyTo}, {})
4285 .addReg(InsertRegs[0], {}, ExtractSubReg);
4289 unsigned LaneIdx = 1;
4290 for (
Register InsReg : InsertRegs) {
4291 Register CopyTo =
I.getOperand(LaneIdx).getReg();
4292 MachineInstr &CopyInst =
4311 I.eraseFromParent();
4315bool AArch64InstructionSelector::selectConcatVectors(
4316 MachineInstr &
I, MachineRegisterInfo &
MRI) {
4317 assert(
I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4318 "Unexpected opcode");
4319 Register Dst =
I.getOperand(0).getReg();
4320 Register Op1 =
I.getOperand(1).getReg();
4321 Register Op2 =
I.getOperand(2).getReg();
4322 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIB);
4325 I.eraseFromParent();
4330AArch64InstructionSelector::emitConstantPoolEntry(
const Constant *CPVal,
4339MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4340 const Constant *CPVal, MachineIRBuilder &MIRBuilder)
const {
4347 RC = &AArch64::FPR128RegClass;
4348 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4351 RC = &AArch64::FPR64RegClass;
4352 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4355 RC = &AArch64::FPR32RegClass;
4356 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4359 RC = &AArch64::FPR16RegClass;
4360 Opc = AArch64::LDRHui;
4363 LLVM_DEBUG(
dbgs() <<
"Could not load from constant pool of type "
4368 MachineInstr *LoadMI =
nullptr;
4369 auto &MF = MIRBuilder.
getMF();
4370 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4371 if (IsTiny && (
Size == 16 ||
Size == 8 ||
Size == 4)) {
4373 LoadMI = &*MIRBuilder.
buildInstr(
Opc, {RC}, {}).addConstantPoolIndex(CPIdx);
4376 MIRBuilder.
buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
4380 .addConstantPoolIndex(
4396static std::pair<unsigned, unsigned>
4398 unsigned Opc, SubregIdx;
4399 if (RB.
getID() == AArch64::GPRRegBankID) {
4401 Opc = AArch64::INSvi8gpr;
4402 SubregIdx = AArch64::bsub;
4403 }
else if (EltSize == 16) {
4404 Opc = AArch64::INSvi16gpr;
4405 SubregIdx = AArch64::ssub;
4406 }
else if (EltSize == 32) {
4407 Opc = AArch64::INSvi32gpr;
4408 SubregIdx = AArch64::ssub;
4409 }
else if (EltSize == 64) {
4410 Opc = AArch64::INSvi64gpr;
4411 SubregIdx = AArch64::dsub;
4417 Opc = AArch64::INSvi8lane;
4418 SubregIdx = AArch64::bsub;
4419 }
else if (EltSize == 16) {
4420 Opc = AArch64::INSvi16lane;
4421 SubregIdx = AArch64::hsub;
4422 }
else if (EltSize == 32) {
4423 Opc = AArch64::INSvi32lane;
4424 SubregIdx = AArch64::ssub;
4425 }
else if (EltSize == 64) {
4426 Opc = AArch64::INSvi64lane;
4427 SubregIdx = AArch64::dsub;
4432 return std::make_pair(
Opc, SubregIdx);
4435MachineInstr *AArch64InstructionSelector::emitInstr(
4436 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4437 std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
4438 const ComplexRendererFns &RenderFns)
const {
4439 assert(Opcode &&
"Expected an opcode?");
4441 "Function should only be used to produce selected instructions!");
4442 auto MI = MIRBuilder.
buildInstr(Opcode, DstOps, SrcOps);
4444 for (
auto &Fn : *RenderFns)
4450MachineInstr *AArch64InstructionSelector::emitAddSub(
4451 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4453 MachineIRBuilder &MIRBuilder)
const {
4455 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4459 assert((
Size == 32 ||
Size == 64) &&
"Expected a 32-bit or 64-bit type only");
4460 bool Is32Bit =
Size == 32;
4463 if (
auto Fns = selectArithImmed(
RHS))
4464 return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {
LHS},
4468 if (
auto Fns = selectNegArithImmed(
RHS))
4469 return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {
LHS},
4473 if (
auto Fns = selectArithExtendedRegister(
RHS))
4474 return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {
LHS},
4478 if (
auto Fns = selectShiftedRegister(
RHS))
4479 return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {
LHS},
4481 return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {
LHS,
RHS},
4486AArch64InstructionSelector::emitADD(
Register DefReg, MachineOperand &
LHS,
4487 MachineOperand &
RHS,
4488 MachineIRBuilder &MIRBuilder)
const {
4489 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4490 {{AArch64::ADDXri, AArch64::ADDWri},
4491 {AArch64::ADDXrs, AArch64::ADDWrs},
4492 {AArch64::ADDXrr, AArch64::ADDWrr},
4493 {AArch64::SUBXri, AArch64::SUBWri},
4494 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4495 return emitAddSub(OpcTable, DefReg,
LHS,
RHS, MIRBuilder);
4499AArch64InstructionSelector::emitADDS(
Register Dst, MachineOperand &
LHS,
4500 MachineOperand &
RHS,
4501 MachineIRBuilder &MIRBuilder)
const {
4502 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4503 {{AArch64::ADDSXri, AArch64::ADDSWri},
4504 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4505 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4506 {AArch64::SUBSXri, AArch64::SUBSWri},
4507 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4508 return emitAddSub(OpcTable, Dst,
LHS,
RHS, MIRBuilder);
4512AArch64InstructionSelector::emitSUBS(
Register Dst, MachineOperand &
LHS,
4513 MachineOperand &
RHS,
4514 MachineIRBuilder &MIRBuilder)
const {
4515 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4516 {{AArch64::SUBSXri, AArch64::SUBSWri},
4517 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4518 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4519 {AArch64::ADDSXri, AArch64::ADDSWri},
4520 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4521 return emitAddSub(OpcTable, Dst,
LHS,
RHS, MIRBuilder);
4525AArch64InstructionSelector::emitADCS(
Register Dst, MachineOperand &
LHS,
4526 MachineOperand &
RHS,
4527 MachineIRBuilder &MIRBuilder)
const {
4528 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4529 MachineRegisterInfo *
MRI = MIRBuilder.
getMRI();
4531 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4532 return emitInstr(OpcTable[Is32Bit], {Dst}, {
LHS,
RHS}, MIRBuilder);
4536AArch64InstructionSelector::emitSBCS(
Register Dst, MachineOperand &
LHS,
4537 MachineOperand &
RHS,
4538 MachineIRBuilder &MIRBuilder)
const {
4539 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4540 MachineRegisterInfo *
MRI = MIRBuilder.
getMRI();
4542 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4543 return emitInstr(OpcTable[Is32Bit], {Dst}, {
LHS,
RHS}, MIRBuilder);
4547AArch64InstructionSelector::emitCMP(MachineOperand &
LHS, MachineOperand &
RHS,
4548 MachineIRBuilder &MIRBuilder)
const {
4551 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4556AArch64InstructionSelector::emitCMN(MachineOperand &
LHS, MachineOperand &
RHS,
4557 MachineIRBuilder &MIRBuilder)
const {
4560 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4565AArch64InstructionSelector::emitTST(MachineOperand &
LHS, MachineOperand &
RHS,
4566 MachineIRBuilder &MIRBuilder)
const {
4567 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4571 bool Is32Bit = (
RegSize == 32);
4572 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4573 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4574 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4578 int64_t
Imm = ValAndVReg->Value.getSExtValue();
4581 auto TstMI = MIRBuilder.
buildInstr(OpcTable[0][Is32Bit], {Ty}, {
LHS});
4588 if (
auto Fns = selectLogicalShiftedRegister(
RHS))
4589 return emitInstr(OpcTable[1][Is32Bit], {Ty}, {
LHS}, MIRBuilder, Fns);
4590 return emitInstr(OpcTable[2][Is32Bit], {Ty}, {
LHS,
RHS}, MIRBuilder);
4593MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4594 MachineOperand &
LHS, MachineOperand &
RHS, MachineOperand &Predicate,
4595 MachineIRBuilder &MIRBuilder)
const {
4596 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected LHS and RHS to be registers!");
4603 assert((
Size == 32 ||
Size == 64) &&
"Expected a 32-bit or 64-bit LHS/RHS?");
4605 if (
auto FoldCmp = tryFoldIntegerCompare(
LHS,
RHS, Predicate, MIRBuilder))
4607 return emitCMP(
LHS,
RHS, MIRBuilder);
4610MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4612 MachineRegisterInfo &
MRI = *MIRBuilder.
getMRI();
4616 "Expected a 32-bit scalar register?");
4618 const Register ZReg = AArch64::WZR;
4623 return emitCSINC(Dst, ZReg, ZReg, InvCC1,
4629 emitCSINC(Def1Reg, ZReg, ZReg, InvCC1, MIRBuilder);
4630 emitCSINC(Def2Reg, ZReg, ZReg, InvCC2, MIRBuilder);
4631 auto OrMI = MIRBuilder.
buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
4636MachineInstr *AArch64InstructionSelector::emitFPCompare(
4638 std::optional<CmpInst::Predicate>
Pred)
const {
4639 MachineRegisterInfo &
MRI = *MIRBuilder.
getMRI();
4644 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4654 if (!ShouldUseImm &&
Pred && IsEqualityPred(*
Pred)) {
4657 ShouldUseImm =
true;
4661 unsigned CmpOpcTbl[2][3] = {
4662 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4663 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4665 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4677MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4679 MachineIRBuilder &MIRBuilder)
const {
4689 if (Op1Ty != Op2Ty) {
4690 LLVM_DEBUG(
dbgs() <<
"Could not do vector concat of differing vector tys");
4693 assert(Op1Ty.
isVector() &&
"Expected a vector for vector concat");
4696 LLVM_DEBUG(
dbgs() <<
"Vector concat not supported for full size vectors");
4711 MachineInstr *WidenedOp1 =
4712 emitScalarToVector(ScalarTy.
getSizeInBits(), DstRC, Op1, MIRBuilder);
4713 MachineInstr *WidenedOp2 =
4714 emitScalarToVector(ScalarTy.
getSizeInBits(), DstRC, Op2, MIRBuilder);
4715 if (!WidenedOp1 || !WidenedOp2) {
4716 LLVM_DEBUG(
dbgs() <<
"Could not emit a vector from scalar value");
4721 unsigned InsertOpc, InsSubRegIdx;
4722 std::tie(InsertOpc, InsSubRegIdx) =
4740 MachineIRBuilder &MIRBuilder)
const {
4747 Size =
TRI.getRegSizeInBits(*RC);
4751 assert(
Size <= 64 &&
"Expected 64 bits or less only!");
4752 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4753 unsigned Opc = OpcTable[
Size == 64];
4759MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &
I,
4761 MachineRegisterInfo *
MRI = MIB.
getMRI();
4762 unsigned Opcode =
I.getOpcode();
4766 bool NeedsNegatedCarry =
4767 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4784 if (SrcMI == PrevMI) {
4786 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4787 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4788 CarrySrcMI->isUnsigned() &&
4789 CarrySrcMI->getCarryOutReg() == CarryReg &&
4790 selectAndRestoreState(*SrcMI))
4797 if (NeedsNegatedCarry) {
4800 return emitInstr(AArch64::SUBSWrr, {DeadReg}, {ZReg, CarryReg}, MIB);
4804 auto Fns = select12BitValueWithLeftShift(1);
4805 return emitInstr(AArch64::SUBSWri, {DeadReg}, {CarryReg}, MIB, Fns);
4808bool AArch64InstructionSelector::selectOverflowOp(MachineInstr &
I,
4809 MachineRegisterInfo &
MRI) {
4814 emitCarryIn(
I, CarryInMI->getCarryInReg());
4818 auto OpAndCC = emitOverflowOp(
I.getOpcode(), CarryMI.getDstReg(),
4819 CarryMI.getLHS(), CarryMI.getRHS(), MIB);
4821 Register CarryOutReg = CarryMI.getCarryOutReg();
4825 OpAndCC.first->addRegisterDead(AArch64::NZCV, &
TRI);
4832 emitCSINC(CarryOutReg, ZReg, ZReg,
4833 getInvertedCondCode(OpAndCC.second), MIB);
4836 I.eraseFromParent();
4840std::pair<MachineInstr *, AArch64CC::CondCode>
4841AArch64InstructionSelector::emitOverflowOp(
unsigned Opcode,
Register Dst,
4842 MachineOperand &
LHS,
4843 MachineOperand &
RHS,
4844 MachineIRBuilder &MIRBuilder)
const {
4848 case TargetOpcode::G_SADDO:
4850 case TargetOpcode::G_UADDO:
4852 case TargetOpcode::G_SSUBO:
4854 case TargetOpcode::G_USUBO:
4856 case TargetOpcode::G_SADDE:
4858 case TargetOpcode::G_UADDE:
4860 case TargetOpcode::G_SSUBE:
4862 case TargetOpcode::G_USUBE:
4883 unsigned Depth = 0) {
4884 if (!
MRI.hasOneNonDBGUse(Val))
4890 MustBeFirst =
false;
4896 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4897 bool IsOR = Opcode == TargetOpcode::G_OR;
4909 if (MustBeFirstL && MustBeFirstR)
4915 if (!CanNegateL && !CanNegateR)
4919 CanNegate = WillNegate && CanNegateL && CanNegateR;
4922 MustBeFirst = !CanNegate;
4924 assert(Opcode == TargetOpcode::G_AND &&
"Must be G_AND");
4927 MustBeFirst = MustBeFirstL || MustBeFirstR;
4934MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
4937 MachineIRBuilder &MIB)
const {
4941 std::optional<ValueAndVReg>
C;
4945 if (!
C ||
C->Value.sgt(31) ||
C->Value.slt(-31))
4946 CCmpOpc = OpTy.
getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
4947 else if (
C->Value.ule(31))
4948 CCmpOpc = OpTy.
getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
4950 CCmpOpc = OpTy.
getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
4956 assert(STI.hasFullFP16() &&
"Expected Full FP16 for fp16 comparisons");
4957 CCmpOpc = AArch64::FCCMPHrr;
4960 CCmpOpc = AArch64::FCCMPSrr;
4963 CCmpOpc = AArch64::FCCMPDrr;
4973 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
4974 CCmp.
addImm(
C->Value.getZExtValue());
4975 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
4976 CCmp.
addImm(
C->Value.abs().getZExtValue());
4984MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
5006 MachineInstr *ExtraCmp;
5008 ExtraCmp = emitFPCompare(
LHS,
RHS, MIB,
CC);
5020 return emitCMP(
Cmp->getOperand(2),
Cmp->getOperand(3), MIB);
5021 return emitFPCompare(
Cmp->getOperand(2).getReg(),
5022 Cmp->getOperand(3).getReg(), MIB);
5029 bool IsOR = Opcode == TargetOpcode::G_OR;
5035 assert(ValidL &&
"Valid conjunction/disjunction tree");
5042 assert(ValidR &&
"Valid conjunction/disjunction tree");
5047 assert(!MustBeFirstR &&
"Valid conjunction/disjunction tree");
5056 bool NegateAfterAll;
5057 if (Opcode == TargetOpcode::G_OR) {
5060 assert(CanNegateR &&
"at least one side must be negatable");
5061 assert(!MustBeFirstR &&
"invalid conjunction/disjunction tree");
5065 NegateAfterR =
true;
5068 NegateR = CanNegateR;
5069 NegateAfterR = !CanNegateR;
5072 NegateAfterAll = !Negate;
5074 assert(Opcode == TargetOpcode::G_AND &&
5075 "Valid conjunction/disjunction tree");
5076 assert(!Negate &&
"Valid conjunction/disjunction tree");
5080 NegateAfterR =
false;
5081 NegateAfterAll =
false;
5086 MachineInstr *CmpR =
5097MachineInstr *AArch64InstructionSelector::emitConjunction(
5099 bool DummyCanNegate;
5100 bool DummyMustBeFirst;
5107bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
5108 MachineInstr &CondMI) {
5119bool AArch64InstructionSelector::tryOptSelect(GSelect &
I) {
5120 MachineRegisterInfo &
MRI = *MIB.
getMRI();
5139 MachineInstr *CondDef =
MRI.
getVRegDef(
I.getOperand(1).getReg());
5148 if (UI.getOpcode() != TargetOpcode::G_SELECT)
5154 unsigned CondOpc = CondDef->
getOpcode();
5155 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
5156 if (tryOptSelectConjunction(
I, *CondDef))
5162 if (CondOpc == TargetOpcode::G_ICMP) {
5191 emitSelect(
I.getOperand(0).getReg(),
I.getOperand(2).getReg(),
5192 I.getOperand(3).getReg(), CondCode, MIB);
5193 I.eraseFromParent();
5197MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
5198 MachineOperand &
LHS, MachineOperand &
RHS, MachineOperand &Predicate,
5199 MachineIRBuilder &MIRBuilder)
const {
5201 "Unexpected MachineOperand");
5202 MachineRegisterInfo &
MRI = *MIRBuilder.
getMRI();
5257 LHSDef->
getOpcode() == TargetOpcode::G_AND) {
5260 if (!ValAndVReg || ValAndVReg->Value != 0)
5270bool AArch64InstructionSelector::selectShuffleVector(
5271 MachineInstr &
I, MachineRegisterInfo &
MRI) {
5272 const LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
5273 Register Src1Reg =
I.getOperand(1).getReg();
5274 Register Src2Reg =
I.getOperand(2).getReg();
5275 ArrayRef<int>
Mask =
I.getOperand(3).getShuffleMask();
5277 "Expected equal shuffle types during selection");
5286 SmallVector<int> NewMask;
5287 bool FirstUsed =
false;
5288 bool SecondUsed =
false;
5289 for (
int M : Mask) {
5291 if (M < 0 || VT->getKnownBits(M < NumElts ? Src1Reg : Src2Reg,
5294 for (
unsigned Byte = 0;
Byte < BytesPerElt; ++
Byte)
5299 FirstUsed |=
M < NumElts;
5300 SecondUsed |=
M >= NumElts;
5301 for (
unsigned Byte = 0;
Byte < BytesPerElt; ++
Byte) {
5310 for (
int &M : NewMask) {
5312 assert(M >= ByteLanes && M < 2 * ByteLanes);
5322 transform(NewMask, std::back_inserter(CstIdxs), [&Ctx](
int M) {
5323 return ConstantInt::get(Type::getInt8Ty(Ctx), M);
5336 emitVectorConcat(std::nullopt, Src1Reg, Src2Reg, MIB);
5343 IndexLoad = emitScalarToVector(64, &AArch64::FPR128RegClass,
5347 AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
5352 MIB.
buildInstr(TargetOpcode::COPY, {
I.getOperand(0).getReg()}, {})
5353 .addReg(TBL1.getReg(0), {}, AArch64::dsub);
5355 I.eraseFromParent();
5360 auto TBL1 = MIB.
buildInstr(AArch64::TBLv16i8One, {
I.getOperand(0)},
5363 I.eraseFromParent();
5371 auto TBL2 = MIB.
buildInstr(AArch64::TBLv16i8Two, {
I.getOperand(0)},
5374 I.eraseFromParent();
5378MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5380 unsigned LaneIdx,
const RegisterBank &RB,
5381 MachineIRBuilder &MIRBuilder)
const {
5382 MachineInstr *InsElt =
nullptr;
5384 MachineRegisterInfo &
MRI = *MIRBuilder.
getMRI();
5393 if (RB.
getID() == AArch64::FPRRegBankID) {
5394 auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
5397 .
addUse(InsSub->getOperand(0).getReg())
5409bool AArch64InstructionSelector::selectUSMovFromExtend(
5410 MachineInstr &
MI, MachineRegisterInfo &
MRI) {
5411 if (
MI.getOpcode() != TargetOpcode::G_SEXT &&
5412 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5413 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5415 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SEXT;
5416 const Register DefReg =
MI.getOperand(0).getReg();
5420 if (DstSize != 32 && DstSize != 64)
5423 MachineInstr *Extract =
getOpcodeDef(TargetOpcode::G_EXTRACT_VECTOR_ELT,
5424 MI.getOperand(1).getReg(),
MRI);
5435 const MachineInstr *ScalarToVector = emitScalarToVector(
5436 VecTy.
getSizeInBits(), &AArch64::FPR128RegClass, Src0, MIB);
5437 assert(ScalarToVector &&
"Didn't expect emitScalarToVector to fail!");
5443 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5445 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5447 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5449 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5451 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5459 MachineInstr *ExtI =
nullptr;
5460 if (DstSize == 64 && !IsSigned) {
5463 ExtI = MIB.
buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
5465 .
addImm(AArch64::sub_32);
5471 MI.eraseFromParent();
5475bool AArch64InstructionSelector::selectIndexedExtLoad(
5476 MachineInstr &
MI, MachineRegisterInfo &
MRI) {
5479 Register WriteBack = ExtLd.getWritebackReg();
5484 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5485 bool IsPre = ExtLd.isPre();
5487 unsigned InsertIntoSubReg = 0;
5493 if ((IsSExt && IsFPR) || Ty.
isVector())
5501 if (MemSizeBits == 8) {
5504 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5506 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5507 NewLdDstTy = IsDst64 ? s64 : s32;
5509 Opc = IsPre ? AArch64::LDRBpre : AArch64::LDRBpost;
5510 InsertIntoSubReg = AArch64::bsub;
5513 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5514 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5517 }
else if (MemSizeBits == 16) {
5520 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5522 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5523 NewLdDstTy = IsDst64 ? s64 : s32;
5525 Opc = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
5526 InsertIntoSubReg = AArch64::hsub;
5529 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5530 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5533 }
else if (MemSizeBits == 32) {
5535 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5538 Opc = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
5539 InsertIntoSubReg = AArch64::ssub;
5542 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5543 InsertIntoSubReg = IsDst64 ? AArch64::sub_32 : 0;
5555 .addImm(Cst->getSExtValue());
5560 if (InsertIntoSubReg) {
5562 auto SubToReg = MIB.
buildInstr(TargetOpcode::SUBREG_TO_REG, {Dst}, {})
5563 .addUse(LdMI.getReg(1))
5564 .
addImm(InsertIntoSubReg);
5574 MI.eraseFromParent();
5579bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &
MI,
5580 MachineRegisterInfo &
MRI) {
5583 Register WriteBack = Ld.getWritebackReg();
5587 "Unexpected type for indexed load");
5588 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5591 return selectIndexedExtLoad(
MI,
MRI);
5595 static constexpr unsigned GPROpcodes[] = {
5596 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5598 static constexpr unsigned FPROpcodes[] = {
5599 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5602 ? FPROpcodes[
Log2_32(MemSize)]
5603 : GPROpcodes[
Log2_32(MemSize)];
5606 static constexpr unsigned GPROpcodes[] = {
5607 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5609 static constexpr unsigned FPROpcodes[] = {
5610 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5611 AArch64::LDRDpost, AArch64::LDRQpost};
5613 ? FPROpcodes[
Log2_32(MemSize)]
5614 : GPROpcodes[
Log2_32(MemSize)];
5624 MI.eraseFromParent();
5628bool AArch64InstructionSelector::selectIndexedStore(GIndexedStore &
I,
5629 MachineRegisterInfo &
MRI) {
5635 "Unexpected type for indexed store");
5637 LocationSize MemSize =
I.getMMO().getSize();
5638 unsigned MemSizeInBytes = MemSize.
getValue();
5640 assert(MemSizeInBytes && MemSizeInBytes <= 16 &&
5641 "Unexpected indexed store size");
5642 unsigned MemSizeLog2 =
Log2_32(MemSizeInBytes);
5646 static constexpr unsigned GPROpcodes[] = {
5647 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5649 static constexpr unsigned FPROpcodes[] = {
5650 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5654 Opc = FPROpcodes[MemSizeLog2];
5656 Opc = GPROpcodes[MemSizeLog2];
5658 static constexpr unsigned GPROpcodes[] = {
5659 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5661 static constexpr unsigned FPROpcodes[] = {
5662 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5663 AArch64::STRDpost, AArch64::STRQpost};
5666 Opc = FPROpcodes[MemSizeLog2];
5668 Opc = GPROpcodes[MemSizeLog2];
5676 Str.cloneMemRefs(
I);
5678 I.eraseFromParent();
5683AArch64InstructionSelector::emitConstantMOVVector(DstOp Dst, APInt
Imm,
5684 MachineRegisterInfo &
MRI) {
5685 unsigned DstSize =
Imm.getBitWidth();
5686 assert((DstSize == 64 || DstSize == 128) &&
5687 "Unexpected vector constant size");
5691 for (AArch64_IMM::ImmInsnModel Insn : Insns) {
5692 switch (Insn.Opcode) {
5693 case AArch64::FMOVD0: {
5697 AArch64::MOVIv2d_ns,
5698 {DstSize == 128 ? Dst : DstOp(&AArch64::FPR128RegClass)}, {})
5701 if (DstSize != 128) {
5702 Mov = MIB.
buildInstr(TargetOpcode::COPY, {Dst}, {})
5703 .addReg(Mov.getReg(0), {}, AArch64::dsub);
5709 case AArch64::FMOVDi:
5710 case AArch64::FMOVv2f32_ns:
5711 case AArch64::FMOVv2f64_ns:
5712 case AArch64::FMOVv4f32_ns:
5713 case AArch64::FMOVv4f16_ns:
5714 case AArch64::FMOVv8f16_ns:
5715 case AArch64::MOVID:
5716 case AArch64::MOVIv16b_ns:
5717 case AArch64::MOVIv2d_ns:
5718 case AArch64::MOVIv2i32:
5719 case AArch64::MOVIv2s_msl:
5720 case AArch64::MOVIv4i16:
5721 case AArch64::MOVIv4i32:
5722 case AArch64::MOVIv4s_msl:
5723 case AArch64::MOVIv8b_ns:
5724 case AArch64::MOVIv8i16:
5725 case AArch64::MVNIv2i32:
5726 case AArch64::MVNIv2s_msl:
5727 case AArch64::MVNIv4i16:
5728 case AArch64::MVNIv4i32:
5729 case AArch64::MVNIv4s_msl:
5730 case AArch64::MVNIv8i16: {
5731 auto Mov = MIB.
buildInstr(Insn.Opcode, {Dst}, {});
5739 case AArch64::DUPM_ZI: {
5740 auto Mov = MIB.
buildInstr(Insn.Opcode, {&AArch64::ZPRRegClass}, {})
5743 Mov = MIB.
buildInstr(TargetOpcode::COPY, {Dst}, {})
5744 .addReg(Mov.getReg(0), {},
5745 DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5747 DstSize == 64 ? AArch64::FPR64RegClass
5748 : AArch64::FPR128RegClass,
5752 case AArch64::DUP_ZI_D: {
5753 auto Mov = MIB.
buildInstr(Insn.Opcode, {&AArch64::ZPRRegClass}, {})
5757 Mov = MIB.
buildInstr(TargetOpcode::COPY, {Dst}, {})
5758 .addReg(Mov.getReg(0), {},
5759 DstSize == 64 ? AArch64::dsub : AArch64::zsub);
5761 DstSize == 64 ? AArch64::FPR64RegClass
5762 : AArch64::FPR128RegClass,
5774MachineInstr *AArch64InstructionSelector::emitConstantVector(
5775 Register Dst, Constant *CV, APInt
Imm, MachineRegisterInfo &
MRI) {
5776 if (
auto *MovI = emitConstantMOVVector(Dst,
Imm,
MRI))
5781 LLVM_DEBUG(
dbgs() <<
"Could not generate cp load for constant vector!");
5791bool AArch64InstructionSelector::tryOptConstantBuildVec(
5792 MachineInstr &
I, LLT DstTy, MachineRegisterInfo &
MRI) {
5793 assert(
I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5795 assert(DstSize <= 128 &&
"Unexpected build_vec type!");
5801 APInt
Imm(DstSize, 0);
5802 for (
unsigned Idx = 1;
Idx <
I.getNumOperands(); ++
Idx) {
5803 Register OpReg =
I.getOperand(Idx).getReg();
5815 std::move(AnyConst->Value)));
5828 if (!emitConstantVector(
I.getOperand(0).getReg(), CV,
Imm,
MRI))
5830 I.eraseFromParent();
5834bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
5835 MachineInstr &
I, MachineRegisterInfo &
MRI) {
5840 Register Dst =
I.getOperand(0).getReg();
5841 Register EltReg =
I.getOperand(1).getReg();
5850 return !getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Op.getReg(), MRI);
5858 getRegClassForTypeOnBank(
MRI.
getType(Dst), DstRB);
5863 auto SubregToReg = MIB.
buildInstr(AArch64::SUBREG_TO_REG, {Dst}, {})
5866 I.eraseFromParent();
5871bool AArch64InstructionSelector::selectBuildVector(MachineInstr &
I,
5872 MachineRegisterInfo &
MRI) {
5873 assert(
I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5876 const LLT DstTy =
MRI.
getType(
I.getOperand(0).getReg());
5877 const LLT EltTy =
MRI.
getType(
I.getOperand(1).getReg());
5880 if (tryOptConstantBuildVec(
I, DstTy,
MRI))
5882 if (tryOptBuildVecToSubregToReg(
I,
MRI))
5885 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
5890 MachineInstr *ScalarToVec =
5892 I.getOperand(1).getReg(), MIB);
5901 MachineInstr *PrevMI = ScalarToVec;
5902 for (
unsigned i = 2, e = DstSize / EltSize + 1; i <
e; ++i) {
5905 Register OpReg =
I.getOperand(i).getReg();
5908 PrevMI = &*emitLaneInsert(std::nullopt, DstVec, OpReg, i - 1, RB, MIB);
5915 if (DstSize < 128) {
5921 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
5926 unsigned SubReg = 0;
5929 if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
5930 LLVM_DEBUG(
dbgs() <<
"Unsupported destination size! (" << DstSize
5936 Register DstReg =
I.getOperand(0).getReg();
5938 MIB.
buildInstr(TargetOpcode::COPY, {DstReg}, {}).addReg(DstVec, {}, SubReg);
5939 MachineOperand &RegOp =
I.getOperand(1);
5959 if (PrevMI == ScalarToVec && DstReg.
isVirtual()) {
5970bool AArch64InstructionSelector::selectVectorLoadIntrinsic(
unsigned Opc,
5973 assert(
I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
5975 assert(NumVecs > 1 && NumVecs < 5 &&
"Only support 2, 3, or 4 vectors");
5980 "Destination must be 64 bits or 128 bits?");
5981 unsigned SubReg =
Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
5982 auto Ptr =
I.getOperand(
I.getNumOperands() - 1).getReg();
5987 Register SelectedLoadDst =
Load->getOperand(0).getReg();
5988 for (
unsigned Idx = 0;
Idx < NumVecs; ++
Idx) {
5989 auto Vec = MIB.
buildInstr(TargetOpcode::COPY, {
I.getOperand(Idx)}, {})
5990 .addReg(SelectedLoadDst, {}, SubReg +
Idx);
5999bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
6000 unsigned Opc,
unsigned NumVecs, MachineInstr &
I) {
6001 assert(
I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
6003 assert(NumVecs > 1 && NumVecs < 5 &&
"Only support 2, 3, or 4 vectors");
6008 auto FirstSrcRegIt =
I.operands_begin() + NumVecs + 1;
6010 std::transform(FirstSrcRegIt, FirstSrcRegIt + NumVecs, Regs.
begin(),
6011 [](
auto MO) { return MO.getReg(); });
6015 return emitScalarToVector(64, &AArch64::FPR128RegClass, Reg, MIB)
6030 .
addImm(LaneNo->getZExtValue())
6034 Register SelectedLoadDst =
Load->getOperand(0).getReg();
6035 unsigned SubReg = AArch64::qsub0;
6036 for (
unsigned Idx = 0;
Idx < NumVecs; ++
Idx) {
6037 auto Vec = MIB.
buildInstr(TargetOpcode::COPY,
6038 {Narrow ? DstOp(&AArch64::FPR128RegClass)
6041 .addReg(SelectedLoadDst, {}, SubReg +
Idx);
6046 !emitNarrowVector(
I.getOperand(Idx).getReg(), WideReg, MIB,
MRI))
6052void AArch64InstructionSelector::selectVectorStoreIntrinsic(MachineInstr &
I,
6055 MachineRegisterInfo &
MRI =
I.getParent()->getParent()->getRegInfo();
6057 Register Ptr =
I.getOperand(1 + NumVecs).getReg();
6060 std::transform(
I.operands_begin() + 1,
I.operands_begin() + 1 + NumVecs,
6061 Regs.
begin(), [](
auto MO) { return MO.getReg(); });
6070bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
6071 MachineInstr &
I,
unsigned NumVecs,
unsigned Opc) {
6072 MachineRegisterInfo &
MRI =
I.getParent()->getParent()->getRegInfo();
6077 std::transform(
I.operands_begin() + 1,
I.operands_begin() + 1 + NumVecs,
6078 Regs.
begin(), [](
auto MO) { return MO.getReg(); });
6082 return emitScalarToVector(64, &AArch64::FPR128RegClass, Reg, MIB)
6092 Register Ptr =
I.getOperand(1 + NumVecs + 1).getReg();
6095 .
addImm(LaneNo->getZExtValue())
6102bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
6103 MachineInstr &
I, MachineRegisterInfo &
MRI) {
6116 case Intrinsic::aarch64_ldxp:
6117 case Intrinsic::aarch64_ldaxp: {
6119 IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
6120 {
I.getOperand(0).getReg(),
I.getOperand(1).getReg()},
6126 case Intrinsic::aarch64_neon_ld1x2: {
6130 Opc = AArch64::LD1Twov8b;
6132 Opc = AArch64::LD1Twov16b;
6134 Opc = AArch64::LD1Twov4h;
6136 Opc = AArch64::LD1Twov8h;
6138 Opc = AArch64::LD1Twov2s;
6140 Opc = AArch64::LD1Twov4s;
6142 Opc = AArch64::LD1Twov2d;
6143 else if (Ty ==
S64 || Ty == P0)
6144 Opc = AArch64::LD1Twov1d;
6147 selectVectorLoadIntrinsic(
Opc, 2,
I);
6150 case Intrinsic::aarch64_neon_ld1x3: {
6154 Opc = AArch64::LD1Threev8b;
6156 Opc = AArch64::LD1Threev16b;
6158 Opc = AArch64::LD1Threev4h;
6160 Opc = AArch64::LD1Threev8h;
6162 Opc = AArch64::LD1Threev2s;
6164 Opc = AArch64::LD1Threev4s;
6166 Opc = AArch64::LD1Threev2d;
6167 else if (Ty ==
S64 || Ty == P0)
6168 Opc = AArch64::LD1Threev1d;
6171 selectVectorLoadIntrinsic(
Opc, 3,
I);
6174 case Intrinsic::aarch64_neon_ld1x4: {
6178 Opc = AArch64::LD1Fourv8b;
6180 Opc = AArch64::LD1Fourv16b;
6182 Opc = AArch64::LD1Fourv4h;
6184 Opc = AArch64::LD1Fourv8h;
6186 Opc = AArch64::LD1Fourv2s;
6188 Opc = AArch64::LD1Fourv4s;
6190 Opc = AArch64::LD1Fourv2d;
6191 else if (Ty ==
S64 || Ty == P0)
6192 Opc = AArch64::LD1Fourv1d;
6195 selectVectorLoadIntrinsic(
Opc, 4,
I);
6198 case Intrinsic::aarch64_neon_ld2: {
6202 Opc = AArch64::LD2Twov8b;
6204 Opc = AArch64::LD2Twov16b;
6206 Opc = AArch64::LD2Twov4h;
6208 Opc = AArch64::LD2Twov8h;
6210 Opc = AArch64::LD2Twov2s;
6212 Opc = AArch64::LD2Twov4s;
6214 Opc = AArch64::LD2Twov2d;
6215 else if (Ty ==
S64 || Ty == P0)
6216 Opc = AArch64::LD1Twov1d;
6219 selectVectorLoadIntrinsic(
Opc, 2,
I);
6222 case Intrinsic::aarch64_neon_ld2lane: {
6226 Opc = AArch64::LD2i8;
6228 Opc = AArch64::LD2i16;
6230 Opc = AArch64::LD2i32;
6233 Opc = AArch64::LD2i64;
6236 if (!selectVectorLoadLaneIntrinsic(
Opc, 2,
I))
6240 case Intrinsic::aarch64_neon_ld2r: {
6244 Opc = AArch64::LD2Rv8b;
6246 Opc = AArch64::LD2Rv16b;
6248 Opc = AArch64::LD2Rv4h;
6250 Opc = AArch64::LD2Rv8h;
6252 Opc = AArch64::LD2Rv2s;
6254 Opc = AArch64::LD2Rv4s;
6256 Opc = AArch64::LD2Rv2d;
6257 else if (Ty ==
S64 || Ty == P0)
6258 Opc = AArch64::LD2Rv1d;
6261 selectVectorLoadIntrinsic(
Opc, 2,
I);
6264 case Intrinsic::aarch64_neon_ld3: {
6268 Opc = AArch64::LD3Threev8b;
6270 Opc = AArch64::LD3Threev16b;
6272 Opc = AArch64::LD3Threev4h;
6274 Opc = AArch64::LD3Threev8h;
6276 Opc = AArch64::LD3Threev2s;
6278 Opc = AArch64::LD3Threev4s;
6280 Opc = AArch64::LD3Threev2d;
6281 else if (Ty ==
S64 || Ty == P0)
6282 Opc = AArch64::LD1Threev1d;
6285 selectVectorLoadIntrinsic(
Opc, 3,
I);
6288 case Intrinsic::aarch64_neon_ld3lane: {
6292 Opc = AArch64::LD3i8;
6294 Opc = AArch64::LD3i16;
6296 Opc = AArch64::LD3i32;
6299 Opc = AArch64::LD3i64;
6302 if (!selectVectorLoadLaneIntrinsic(
Opc, 3,
I))
6306 case Intrinsic::aarch64_neon_ld3r: {
6310 Opc = AArch64::LD3Rv8b;
6312 Opc = AArch64::LD3Rv16b;
6314 Opc = AArch64::LD3Rv4h;
6316 Opc = AArch64::LD3Rv8h;
6318 Opc = AArch64::LD3Rv2s;
6320 Opc = AArch64::LD3Rv4s;
6322 Opc = AArch64::LD3Rv2d;
6323 else if (Ty ==
S64 || Ty == P0)
6324 Opc = AArch64::LD3Rv1d;
6327 selectVectorLoadIntrinsic(
Opc, 3,
I);
6330 case Intrinsic::aarch64_neon_ld4: {
6334 Opc = AArch64::LD4Fourv8b;
6336 Opc = AArch64::LD4Fourv16b;
6338 Opc = AArch64::LD4Fourv4h;
6340 Opc = AArch64::LD4Fourv8h;
6342 Opc = AArch64::LD4Fourv2s;
6344 Opc = AArch64::LD4Fourv4s;
6346 Opc = AArch64::LD4Fourv2d;
6347 else if (Ty ==
S64 || Ty == P0)
6348 Opc = AArch64::LD1Fourv1d;
6351 selectVectorLoadIntrinsic(
Opc, 4,
I);
6354 case Intrinsic::aarch64_neon_ld4lane: {
6358 Opc = AArch64::LD4i8;
6360 Opc = AArch64::LD4i16;
6362 Opc = AArch64::LD4i32;
6365 Opc = AArch64::LD4i64;
6368 if (!selectVectorLoadLaneIntrinsic(
Opc, 4,
I))
6372 case Intrinsic::aarch64_neon_ld4r: {
6376 Opc = AArch64::LD4Rv8b;
6378 Opc = AArch64::LD4Rv16b;
6380 Opc = AArch64::LD4Rv4h;
6382 Opc = AArch64::LD4Rv8h;
6384 Opc = AArch64::LD4Rv2s;
6386 Opc = AArch64::LD4Rv4s;
6388 Opc = AArch64::LD4Rv2d;
6389 else if (Ty ==
S64 || Ty == P0)
6390 Opc = AArch64::LD4Rv1d;
6393 selectVectorLoadIntrinsic(
Opc, 4,
I);
6396 case Intrinsic::aarch64_neon_st1x2: {
6400 Opc = AArch64::ST1Twov8b;
6402 Opc = AArch64::ST1Twov16b;
6404 Opc = AArch64::ST1Twov4h;
6406 Opc = AArch64::ST1Twov8h;
6408 Opc = AArch64::ST1Twov2s;
6410 Opc = AArch64::ST1Twov4s;
6412 Opc = AArch64::ST1Twov2d;
6413 else if (Ty ==
S64 || Ty == P0)
6414 Opc = AArch64::ST1Twov1d;
6417 selectVectorStoreIntrinsic(
I, 2,
Opc);
6420 case Intrinsic::aarch64_neon_st1x3: {
6424 Opc = AArch64::ST1Threev8b;
6426 Opc = AArch64::ST1Threev16b;
6428 Opc = AArch64::ST1Threev4h;
6430 Opc = AArch64::ST1Threev8h;
6432 Opc = AArch64::ST1Threev2s;
6434 Opc = AArch64::ST1Threev4s;
6436 Opc = AArch64::ST1Threev2d;
6437 else if (Ty ==
S64 || Ty == P0)
6438 Opc = AArch64::ST1Threev1d;
6441 selectVectorStoreIntrinsic(
I, 3,
Opc);
6444 case Intrinsic::aarch64_neon_st1x4: {
6448 Opc = AArch64::ST1Fourv8b;
6450 Opc = AArch64::ST1Fourv16b;
6452 Opc = AArch64::ST1Fourv4h;
6454 Opc = AArch64::ST1Fourv8h;
6456 Opc = AArch64::ST1Fourv2s;
6458 Opc = AArch64::ST1Fourv4s;
6460 Opc = AArch64::ST1Fourv2d;
6461 else if (Ty ==
S64 || Ty == P0)
6462 Opc = AArch64::ST1Fourv1d;
6465 selectVectorStoreIntrinsic(
I, 4,
Opc);
6468 case Intrinsic::aarch64_neon_st2: {
6472 Opc = AArch64::ST2Twov8b;
6474 Opc = AArch64::ST2Twov16b;
6476 Opc = AArch64::ST2Twov4h;
6478 Opc = AArch64::ST2Twov8h;
6480 Opc = AArch64::ST2Twov2s;
6482 Opc = AArch64::ST2Twov4s;
6484 Opc = AArch64::ST2Twov2d;
6485 else if (Ty ==
S64 || Ty == P0)
6486 Opc = AArch64::ST1Twov1d;
6489 selectVectorStoreIntrinsic(
I, 2,
Opc);
6492 case Intrinsic::aarch64_neon_st3: {
6496 Opc = AArch64::ST3Threev8b;
6498 Opc = AArch64::ST3Threev16b;
6500 Opc = AArch64::ST3Threev4h;
6502 Opc = AArch64::ST3Threev8h;
6504 Opc = AArch64::ST3Threev2s;
6506 Opc = AArch64::ST3Threev4s;
6508 Opc = AArch64::ST3Threev2d;
6509 else if (Ty ==
S64 || Ty == P0)
6510 Opc = AArch64::ST1Threev1d;
6513 selectVectorStoreIntrinsic(
I, 3,
Opc);
6516 case Intrinsic::aarch64_neon_st4: {
6520 Opc = AArch64::ST4Fourv8b;
6522 Opc = AArch64::ST4Fourv16b;
6524 Opc = AArch64::ST4Fourv4h;
6526 Opc = AArch64::ST4Fourv8h;
6528 Opc = AArch64::ST4Fourv2s;
6530 Opc = AArch64::ST4Fourv4s;
6532 Opc = AArch64::ST4Fourv2d;
6533 else if (Ty ==
S64 || Ty == P0)
6534 Opc = AArch64::ST1Fourv1d;
6537 selectVectorStoreIntrinsic(
I, 4,
Opc);
6540 case Intrinsic::aarch64_neon_st2lane: {
6544 Opc = AArch64::ST2i8;
6546 Opc = AArch64::ST2i16;
6548 Opc = AArch64::ST2i32;
6551 Opc = AArch64::ST2i64;
6554 if (!selectVectorStoreLaneIntrinsic(
I, 2,
Opc))
6558 case Intrinsic::aarch64_neon_st3lane: {
6562 Opc = AArch64::ST3i8;
6564 Opc = AArch64::ST3i16;
6566 Opc = AArch64::ST3i32;
6569 Opc = AArch64::ST3i64;
6572 if (!selectVectorStoreLaneIntrinsic(
I, 3,
Opc))
6576 case Intrinsic::aarch64_neon_st4lane: {
6580 Opc = AArch64::ST4i8;
6582 Opc = AArch64::ST4i16;
6584 Opc = AArch64::ST4i32;
6587 Opc = AArch64::ST4i64;
6590 if (!selectVectorStoreLaneIntrinsic(
I, 4,
Opc))
6594 case Intrinsic::aarch64_mops_memset_tag: {
6607 Register DstDef =
I.getOperand(0).getReg();
6609 Register DstUse =
I.getOperand(2).getReg();
6610 Register ValUse =
I.getOperand(3).getReg();
6611 Register SizeUse =
I.getOperand(4).getReg();
6618 auto Memset = MIB.
buildInstr(AArch64::MOPSMemorySetTaggingPseudo,
6619 {DstDef, SizeDef}, {DstUse, SizeUse, ValUse});
6625 case Intrinsic::ptrauth_resign_load_relative: {
6626 Register DstReg =
I.getOperand(0).getReg();
6627 Register ValReg =
I.getOperand(2).getReg();
6628 uint64_t AUTKey =
I.getOperand(3).getImm();
6629 Register AUTDisc =
I.getOperand(4).getReg();
6630 uint64_t PACKey =
I.getOperand(5).getImm();
6631 Register PACDisc =
I.getOperand(6).getReg();
6632 int64_t Addend =
I.getOperand(7).getImm();
6635 uint16_t AUTConstDiscC = 0;
6636 std::tie(AUTConstDiscC, AUTAddrDisc) =
6640 uint16_t PACConstDiscC = 0;
6641 std::tie(PACConstDiscC, PACAddrDisc) =
6644 MIB.
buildCopy({AArch64::X16}, {ValReg});
6660 I.eraseFromParent();
6665 I.eraseFromParent();
6669bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &
I,
6670 MachineRegisterInfo &
MRI) {
6676 case Intrinsic::ptrauth_resign: {
6677 Register DstReg =
I.getOperand(0).getReg();
6678 Register ValReg =
I.getOperand(2).getReg();
6679 uint64_t AUTKey =
I.getOperand(3).getImm();
6680 Register AUTDisc =
I.getOperand(4).getReg();
6681 uint64_t PACKey =
I.getOperand(5).getImm();
6682 Register PACDisc =
I.getOperand(6).getReg();
6685 uint16_t AUTConstDiscC = 0;
6686 std::tie(AUTConstDiscC, AUTAddrDisc) =
6690 uint16_t PACConstDiscC = 0;
6691 std::tie(PACConstDiscC, PACAddrDisc) =
6694 MIB.
buildCopy({AArch64::X16}, {ValReg});
6708 I.eraseFromParent();
6711 case Intrinsic::ptrauth_auth_with_pc_and_resign: {
6712 Register DstReg =
I.getOperand(0).getReg();
6713 Register ValReg =
I.getOperand(2).getReg();
6714 uint64_t AUTKey =
I.getOperand(3).getImm();
6715 Register AUTDisc =
I.getOperand(4).getReg();
6716 Register AUTPC =
I.getOperand(5).getReg();
6717 uint64_t PACKey =
I.getOperand(6).getImm();
6718 Register PACDisc =
I.getOperand(7).getReg();
6721 "auth_with_pc_and_resign only supports IA and IB keys");
6723 uint16_t PACConstDiscC = 0;
6725 std::tie(PACConstDiscC, PACAddrDisc) =
6729 PACAddrDisc = AArch64::XZR;
6731 MIB.
buildCopy({AArch64::X17}, {ValReg});
6732 MIB.
buildCopy({AArch64::X16}, {AUTDisc});
6747 I.eraseFromParent();
6750 case Intrinsic::ptrauth_auth: {
6751 Register DstReg =
I.getOperand(0).getReg();
6752 Register ValReg =
I.getOperand(2).getReg();
6753 uint64_t AUTKey =
I.getOperand(3).getImm();
6754 Register AUTDisc =
I.getOperand(4).getReg();
6757 uint16_t AUTConstDiscC = 0;
6758 std::tie(AUTConstDiscC, AUTAddrDisc) =
6762 MIB.
buildCopy({AArch64::X16}, {ValReg});
6782 Auth.constrainAllUses(
TII,
TRI, RBI);
6786 I.eraseFromParent();
6789 case Intrinsic::frameaddress:
6790 case Intrinsic::returnaddress: {
6794 unsigned Depth =
I.getOperand(2).getImm();
6795 Register DstReg =
I.getOperand(0).getReg();
6798 if (
Depth == 0 && IntrinID == Intrinsic::returnaddress) {
6799 if (!MFReturnAddr) {
6804 MF,
TII, AArch64::LR, AArch64::GPR64RegClass,
I.getDebugLoc());
6807 if (STI.hasPAuth()) {
6808 MIB.
buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
6815 I.eraseFromParent();
6824 MIB.
buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr}).addImm(0);
6826 FrameAddr = NextFrame;
6829 if (IntrinID == Intrinsic::frameaddress)
6834 if (STI.hasPAuth()) {
6836 MIB.
buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
6837 MIB.
buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
6846 I.eraseFromParent();
6849 case Intrinsic::aarch64_neon_tbl2:
6850 SelectTable(
I,
MRI, 2, AArch64::TBLv8i8Two, AArch64::TBLv16i8Two,
false);
6852 case Intrinsic::aarch64_neon_tbl3:
6853 SelectTable(
I,
MRI, 3, AArch64::TBLv8i8Three, AArch64::TBLv16i8Three,
6856 case Intrinsic::aarch64_neon_tbl4:
6857 SelectTable(
I,
MRI, 4, AArch64::TBLv8i8Four, AArch64::TBLv16i8Four,
false);
6859 case Intrinsic::aarch64_neon_tbx2:
6860 SelectTable(
I,
MRI, 2, AArch64::TBXv8i8Two, AArch64::TBXv16i8Two,
true);
6862 case Intrinsic::aarch64_neon_tbx3:
6863 SelectTable(
I,
MRI, 3, AArch64::TBXv8i8Three, AArch64::TBXv16i8Three,
true);
6865 case Intrinsic::aarch64_neon_tbx4:
6866 SelectTable(
I,
MRI, 4, AArch64::TBXv8i8Four, AArch64::TBXv16i8Four,
true);
6868 case Intrinsic::swift_async_context_addr:
6869 auto Sub = MIB.
buildInstr(AArch64::SUBXri, {
I.getOperand(0).getReg()},
6876 MF->
getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(
true);
6877 I.eraseFromParent();
6912bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
6913 MachineInstr &
I, MachineRegisterInfo &
MRI)
const {
6914 Register DefReg =
I.getOperand(0).getReg();
6915 Register Addr =
I.getOperand(1).getReg();
6917 Register AddrDisc =
I.getOperand(3).getReg();
6918 uint64_t Disc =
I.getOperand(4).getImm();
6928 "constant discriminator in ptrauth global out of range [0, 0xffff]");
6944 if (OffsetMI.
getOpcode() != TargetOpcode::G_CONSTANT)
6956 const GlobalValue *GV;
6967 MachineIRBuilder MIB(
I);
6973 "unsupported non-GOT op flags on ptrauth global reference");
6975 "unsupported non-GOT reference to weak ptrauth global");
6978 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
6985 MIB.
buildInstr(NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
6988 .
addReg(HasAddrDisc ? AddrDisc : AArch64::XZR)
6993 I.eraseFromParent();
7005 "unsupported non-zero offset in weak ptrauth global reference");
7010 MIB.
buildInstr(AArch64::LOADauthptrstatic, {DefReg}, {})
7011 .addGlobalAddress(GV,
Offset)
7016 I.eraseFromParent();
7020void AArch64InstructionSelector::SelectTable(MachineInstr &
I,
7021 MachineRegisterInfo &
MRI,
7022 unsigned NumVec,
unsigned Opc1,
7023 unsigned Opc2,
bool isExt) {
7024 Register DstReg =
I.getOperand(0).getReg();
7029 for (
unsigned i = 0; i < NumVec; i++)
7030 Regs.
push_back(
I.getOperand(i + 2 + isExt).getReg());
7033 Register IdxReg =
I.getOperand(2 + NumVec + isExt).getReg();
7034 MachineInstrBuilder
Instr;
7041 I.eraseFromParent();
7044InstructionSelector::ComplexRendererFns
7045AArch64InstructionSelector::selectShiftA_32(
const MachineOperand &Root)
const {
7047 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7048 return std::nullopt;
7049 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
7050 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7053InstructionSelector::ComplexRendererFns
7054AArch64InstructionSelector::selectShiftB_32(
const MachineOperand &Root)
const {
7056 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
7057 return std::nullopt;
7059 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7062InstructionSelector::ComplexRendererFns
7063AArch64InstructionSelector::selectShiftA_64(
const MachineOperand &Root)
const {
7065 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7066 return std::nullopt;
7067 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
7068 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7071InstructionSelector::ComplexRendererFns
7072AArch64InstructionSelector::selectShiftB_64(
const MachineOperand &Root)
const {
7074 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
7075 return std::nullopt;
7077 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
7080template <
unsigned ShiftW
idth>
7081InstructionSelector::ComplexRendererFns
7082AArch64InstructionSelector::selectShiftMask(MachineOperand &Root)
const {
7084 return std::nullopt;
7086 MachineRegisterInfo &
MRI =
7093 if (ShiftWidth == 32) {
7096 ShAmtReg = ZExtSrcReg;
7105 ShAmtReg = AndSrcReg;
7117 (AddImm % ShiftWidth == 0)) {
7118 ShAmtReg = AddSrcReg;
7119 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
7128 SubImm != 0 && (SubImm % ShiftWidth == 0)) {
7129 return {{[=](MachineInstrBuilder &MIB) {
7130 MachineInstr *
I = MIB.getInstr();
7133 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7134 unsigned SubOpc = ShiftWidth == 32 ? AArch64::SUBWrr : AArch64::SUBXrr;
7135 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7137 auto NegMI =
BuildMI(*
I->getParent(), *
I,
I->getDebugLoc(),
7138 TII.get(SubOpc), NegReg)
7152 (NotImm % ShiftWidth == ShiftWidth - 1)) {
7153 return {{[=](MachineInstrBuilder &MIB) {
7154 MachineInstr *
I = MIB.getInstr();
7157 ShiftWidth == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
7158 unsigned NotOpc = ShiftWidth == 32 ? AArch64::ORNWrr : AArch64::ORNXrr;
7159 Register ZeroReg = ShiftWidth == 32 ? AArch64::WZR : AArch64::XZR;
7161 auto NotMI =
BuildMI(*
I->getParent(), *
I,
I->getDebugLoc(),
7162 TII.get(NotOpc), NotReg)
7172 if (ShAmtReg == Root.
getReg())
7173 return std::nullopt;
7175 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
7183InstructionSelector::ComplexRendererFns
7184AArch64InstructionSelector::select12BitValueWithLeftShift(
7187 if (Immed >> 12 == 0) {
7189 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
7191 Immed = Immed >> 12;
7193 return std::nullopt;
7197 [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
7198 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
7205InstructionSelector::ComplexRendererFns
7206AArch64InstructionSelector::selectArithImmed(MachineOperand &Root)
const {
7213 if (MaybeImmed == std::nullopt)
7214 return std::nullopt;
7215 return select12BitValueWithLeftShift(*MaybeImmed);
7220InstructionSelector::ComplexRendererFns
7221AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root)
const {
7225 return std::nullopt;
7227 if (MaybeImmed == std::nullopt)
7228 return std::nullopt;
7235 return std::nullopt;
7241 Immed = ~((uint32_t)Immed) + 1;
7243 Immed = ~Immed + 1ULL;
7245 if (Immed & 0xFFFFFFFFFF000000ULL)
7246 return std::nullopt;
7248 Immed &= 0xFFFFFFULL;
7249 return select12BitValueWithLeftShift(Immed);
7266std::optional<bool> AArch64InstructionSelector::isWorthFoldingIntoAddrMode(
7267 const MachineInstr &
MI,
const MachineRegisterInfo &
MRI)
const {
7268 if (
MI.getOpcode() == AArch64::G_SHL) {
7272 MI.getOperand(2).getReg(),
MRI)) {
7273 const APInt ShiftVal = ValAndVeg->Value;
7276 return !(STI.hasAddrLSLSlow14() && (ShiftVal == 1 || ShiftVal == 4));
7279 return std::nullopt;
7287bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
7288 const MachineInstr &
MI,
const MachineRegisterInfo &
MRI,
7289 bool IsAddrOperand)
const {
7294 MI.getParent()->getParent()->getFunction().hasOptSize())
7297 if (IsAddrOperand) {
7299 if (
const auto Worth = isWorthFoldingIntoAddrMode(
MI,
MRI))
7303 if (
MI.getOpcode() == AArch64::G_PTR_ADD) {
7304 MachineInstr *OffsetInst =
7310 if (
const auto Worth = isWorthFoldingIntoAddrMode(*OffsetInst,
MRI))
7321 [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
7324InstructionSelector::ComplexRendererFns
7325AArch64InstructionSelector::selectExtendedSHL(
7326 MachineOperand &Root, MachineOperand &
Base, MachineOperand &
Offset,
7327 unsigned SizeInBytes,
bool WantsExt)
const {
7328 assert(
Base.isReg() &&
"Expected base to be a register operand");
7329 assert(
Offset.isReg() &&
"Expected offset to be a register operand");
7334 unsigned OffsetOpc = OffsetInst->
getOpcode();
7335 bool LookedThroughZExt =
false;
7336 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
7338 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
7339 return std::nullopt;
7343 LookedThroughZExt =
true;
7345 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
7346 return std::nullopt;
7349 int64_t LegalShiftVal =
Log2_32(SizeInBytes);
7350 if (LegalShiftVal == 0)
7351 return std::nullopt;
7352 if (!isWorthFoldingIntoExtendedReg(*OffsetInst,
MRI,
true))
7353 return std::nullopt;
7364 if (OffsetOpc == TargetOpcode::G_SHL)
7365 return std::nullopt;
7371 return std::nullopt;
7376 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
7380 if (OffsetOpc == TargetOpcode::G_MUL) {
7382 return std::nullopt;
7388 if ((ImmVal & 0x7) != ImmVal)
7389 return std::nullopt;
7393 if (ImmVal != LegalShiftVal)
7394 return std::nullopt;
7396 unsigned SignExtend = 0;
7400 if (!LookedThroughZExt) {
7402 auto Ext = getExtendTypeForInst(*ExtInst,
MRI,
true);
7404 return std::nullopt;
7409 return std::nullopt;
7415 OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
7420 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(
Base.getReg()); },
7421 [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
7422 [=](MachineInstrBuilder &MIB) {
7425 MIB.addImm(SignExtend);
7438InstructionSelector::ComplexRendererFns
7439AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7440 MachineOperand &Root,
unsigned SizeInBytes)
const {
7442 return std::nullopt;
7457 MachineInstr *PtrAdd =
7459 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd,
MRI,
true))
7460 return std::nullopt;
7464 MachineInstr *OffsetInst =
7466 return selectExtendedSHL(Root, PtrAdd->
getOperand(1),
7479InstructionSelector::ComplexRendererFns
7480AArch64InstructionSelector::selectAddrModeRegisterOffset(
7481 MachineOperand &Root)
const {
7487 return std::nullopt;
7493 return std::nullopt;
7496 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(
Base); },
7497 [=](MachineInstrBuilder &MIB) { MIB.addUse(
Offset); },
7498 [=](MachineInstrBuilder &MIB) {
7508InstructionSelector::ComplexRendererFns
7509AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
7510 unsigned SizeInBytes)
const {
7513 return std::nullopt;
7514 MachineInstr *PtrAdd =
7517 return std::nullopt;
7535 unsigned Scale =
Log2_32(SizeInBytes);
7536 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7540 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7541 ImmOff < (0x1000 << Scale))
7542 return std::nullopt;
7547 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7551 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7557 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7558 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7563 return std::nullopt;
7567 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7573 return selectAddrModeRegisterOffset(Root);
7582InstructionSelector::ComplexRendererFns
7583AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
7584 unsigned SizeInBytes)
const {
7587 MachineInstr *PtrAdd =
7589 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd,
MRI,
true))
7590 return std::nullopt;
7611 auto ExtendedShl = selectExtendedSHL(Root,
LHS, OffsetInst->
getOperand(0),
7620 if (!isWorthFoldingIntoExtendedReg(*OffsetInst,
MRI,
true))
7621 return std::nullopt;
7625 getExtendTypeForInst(*OffsetInst,
MRI,
true);
7627 return std::nullopt;
7630 MachineIRBuilder MIB(*PtrAdd);
7632 AArch64::GPR32RegClass, MIB);
7636 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(
LHS.getReg()); },
7637 [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
7638 [=](MachineInstrBuilder &MIB) {
7639 MIB.addImm(SignExtend);
7649InstructionSelector::ComplexRendererFns
7650AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
7651 unsigned Size)
const {
7652 MachineRegisterInfo &
MRI =
7656 return std::nullopt;
7658 if (!isBaseWithConstantOffset(Root,
MRI))
7659 return std::nullopt;
7663 MachineOperand &OffImm = RootDef->
getOperand(2);
7664 if (!OffImm.
isReg())
7665 return std::nullopt;
7667 if (
RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7668 return std::nullopt;
7670 MachineOperand &RHSOp1 =
RHS->getOperand(1);
7672 return std::nullopt;
7675 if (RHSC >= -256 && RHSC < 256) {
7678 [=](MachineInstrBuilder &MIB) { MIB.add(
Base); },
7679 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
7682 return std::nullopt;
7685InstructionSelector::ComplexRendererFns
7686AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
7688 MachineRegisterInfo &
MRI)
const {
7689 if (RootDef.
getOpcode() != AArch64::G_ADD_LOW)
7690 return std::nullopt;
7693 return std::nullopt;
7698 return std::nullopt;
7702 return std::nullopt;
7706 return std::nullopt;
7710 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
7711 [=](MachineInstrBuilder &MIB) {
7712 MIB.addGlobalAddress(GV,
Offset,
7721InstructionSelector::ComplexRendererFns
7722AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
7723 unsigned Size)
const {
7728 return std::nullopt;
7731 if (RootDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7733 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->
getOperand(1)); },
7734 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
7742 MachineInstr *RootParent = Root.
getParent();
7744 !(RootParent->
getOpcode() == AArch64::G_AARCH64_PREFETCH &&
7746 auto OpFns = tryFoldAddLowIntoImm(*RootDef,
Size,
MRI);
7751 if (isBaseWithConstantOffset(Root,
MRI)) {
7759 if ((RHSC & (
Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7760 if (LHSDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX)
7762 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->
getOperand(1)); },
7763 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
7767 [=](MachineInstrBuilder &MIB) { MIB.add(
LHS); },
7768 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
7775 if (selectAddrModeUnscaled(Root,
Size))
7776 return std::nullopt;
7779 [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
7780 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
7787 switch (
MI.getOpcode()) {
7790 case TargetOpcode::G_SHL:
7792 case TargetOpcode::G_LSHR:
7794 case TargetOpcode::G_ASHR:
7796 case TargetOpcode::G_ROTR:
7803InstructionSelector::ComplexRendererFns
7804AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
7805 bool AllowROR)
const {
7807 return std::nullopt;
7808 MachineRegisterInfo &
MRI =
7816 return std::nullopt;
7818 return std::nullopt;
7819 if (!isWorthFoldingIntoExtendedReg(*ShiftInst,
MRI,
false))
7820 return std::nullopt;
7823 MachineOperand &ShiftRHS = ShiftInst->
getOperand(2);
7826 return std::nullopt;
7830 MachineOperand &ShiftLHS = ShiftInst->
getOperand(1);
7834 unsigned Val = *Immed & (NumBits - 1);
7837 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); },
7838 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}};
7842 MachineInstr &
MI, MachineRegisterInfo &
MRI,
bool IsLoadStore)
const {
7843 unsigned Opc =
MI.getOpcode();
7846 if (
Opc == TargetOpcode::G_SEXT ||
Opc == TargetOpcode::G_SEXT_INREG) {
7848 if (
Opc == TargetOpcode::G_SEXT)
7851 Size =
MI.getOperand(2).getImm();
7852 assert(
Size != 64 &&
"Extend from 64 bits?");
7865 if (
Opc == TargetOpcode::G_ZEXT ||
Opc == TargetOpcode::G_ANYEXT) {
7867 assert(
Size != 64 &&
"Extend from 64 bits?");
7882 if (
Opc != TargetOpcode::G_AND)
7901Register AArch64InstructionSelector::moveScalarRegClass(
7903 MachineRegisterInfo &
MRI = *MIB.
getMRI();
7913 return Copy.getReg(0);
7918InstructionSelector::ComplexRendererFns
7919AArch64InstructionSelector::selectArithExtendedRegister(
7920 MachineOperand &Root)
const {
7922 return std::nullopt;
7923 MachineRegisterInfo &
MRI =
7931 return std::nullopt;
7933 if (!isWorthFoldingIntoExtendedReg(*RootDef,
MRI,
false))
7934 return std::nullopt;
7937 if (RootDef->
getOpcode() == TargetOpcode::G_SHL) {
7942 return std::nullopt;
7943 ShiftVal = *MaybeShiftVal;
7945 return std::nullopt;
7950 return std::nullopt;
7951 Ext = getExtendTypeForInst(*ExtDef,
MRI);
7953 return std::nullopt;
7957 Ext = getExtendTypeForInst(*RootDef,
MRI);
7959 return std::nullopt;
7968 if (isDef32(*ExtInst))
7969 return std::nullopt;
7975 MachineIRBuilder MIB(*RootDef);
7976 ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);
7978 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
7979 [=](MachineInstrBuilder &MIB) {
7980 MIB.addImm(getArithExtendImm(Ext, ShiftVal));
7984InstructionSelector::ComplexRendererFns
7985AArch64InstructionSelector::selectExtractHigh(MachineOperand &Root)
const {
7987 return std::nullopt;
7988 MachineRegisterInfo &
MRI =
7992 while (Extract && Extract->MI->
getOpcode() == TargetOpcode::G_BITCAST &&
7997 return std::nullopt;
8000 if (Unmerge->getNumDefs() == 2 &&
8002 Register ExtReg = Unmerge->getSourceReg();
8003 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }}};
8007 LLT SrcTy =
MRI.
getType(ExtElt->getVectorReg());
8011 LaneIdx->Value.getSExtValue() == 1) {
8012 Register ExtReg = ExtElt->getVectorReg();
8013 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }}};
8017 LLT SrcTy =
MRI.
getType(Subvec->getSrcVec());
8018 auto LaneIdx = Subvec->getIndexImm();
8020 Register ExtReg = Subvec->getSrcVec();
8021 return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); }}};
8025 return std::nullopt;
8028InstructionSelector::ComplexRendererFns
8029AArch64InstructionSelector::selectCVTFixedPointBase(
const MachineOperand &Root,
8030 unsigned DstElemWidth,
8031 bool isReciprocal)
const {
8033 return std::nullopt;
8034 const MachineRegisterInfo &
MRI =
8040 if (Dup && Dup->
getOpcode() == TargetOpcode::G_SPLAT_VECTOR)
8043 std::optional<ValueAndVReg> CstVal =
8047 return std::nullopt;
8051 switch (CstElemWidth) {
8053 FVal =
APFloat(APFloat::IEEEhalf(), CstVal->Value.trunc(CstElemWidth));
8056 FVal =
APFloat(APFloat::IEEEsingle(), CstVal->Value);
8059 FVal =
APFloat(APFloat::IEEEdouble(), CstVal->Value);
8062 return std::nullopt;
8064 if (
unsigned FBits =
8066 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(FBits); }}};
8068 return std::nullopt;
8071unsigned AArch64InstructionSelector::getFixedPointWidthFromOperand(
8072 const MachineOperand &Root)
const {
8080template <
unsigned W
idth>
8081InstructionSelector::ComplexRendererFns
8082AArch64InstructionSelector::selectCVTFixedPoint(MachineOperand &Root)
const {
8083 return selectCVTFixedPointBase(Root, Width,
false);
8086template <
unsigned W
idth>
8087InstructionSelector::ComplexRendererFns
8088AArch64InstructionSelector::selectCVTFixedPosRecipOperand(
8089 MachineOperand &Root)
const {
8090 return selectCVTFixedPointBase(Root, Width,
true);
8093InstructionSelector::ComplexRendererFns
8094AArch64InstructionSelector::selectCVTFixedPointVec(MachineOperand &Root)
const {
8095 return selectCVTFixedPointBase(Root, getFixedPointWidthFromOperand(Root),
8099InstructionSelector::ComplexRendererFns
8100AArch64InstructionSelector::selectCVTFixedPosRecipOperandVec(
8101 MachineOperand &Root)
const {
8102 return selectCVTFixedPointBase(Root, getFixedPointWidthFromOperand(Root),
8106void AArch64InstructionSelector::renderFixedPointScalarXForm(
8107 MachineInstrBuilder &MIB,
const MachineInstr &
MI,
int OpIdx)
const {
8108 assert(OpIdx == 3 &&
MI.getOperand(OpIdx).isImm() &&
8109 "Expected vecshift immediate operand");
8110 MIB.
addImm(
MI.getOperand(OpIdx).getImm());
8113void AArch64InstructionSelector::renderFixedPointImm(MachineInstrBuilder &MIB,
8114 const MachineOperand &Root,
8116 bool isReciprocal)
const {
8120 InstructionSelector::ComplexRendererFns Renderer =
8121 selectCVTFixedPointBase(Root, Width, isReciprocal);
8122 assert((Renderer && Renderer->size() == 1) &&
8123 "Expected selectCVTFixedPointBase to provide a function\n");
8124 (Renderer->front())(MIB);
8127void AArch64InstructionSelector::renderFixedPointXForm(MachineInstrBuilder &MIB,
8128 const MachineInstr &
MI,
8130 const MachineOperand &Root =
MI.getOperand(OpIdx);
8131 renderFixedPointImm(MIB, Root, getFixedPointWidthFromOperand(Root),
8135void AArch64InstructionSelector::renderFixedPointRecipXForm(
8136 MachineInstrBuilder &MIB,
const MachineInstr &
MI,
int OpIdx)
const {
8137 const MachineOperand &Root =
MI.getOperand(OpIdx);
8138 renderFixedPointImm(MIB, Root, getFixedPointWidthFromOperand(Root),
8142void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
8143 const MachineInstr &
MI,
8145 const MachineRegisterInfo &
MRI =
MI.getParent()->getParent()->getRegInfo();
8146 assert(
MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8147 "Expected G_CONSTANT");
8148 std::optional<int64_t> CstVal =
8150 assert(CstVal &&
"Expected constant value");
8154void AArch64InstructionSelector::renderLogicalImm32(
8155 MachineInstrBuilder &MIB,
const MachineInstr &
I,
int OpIdx)
const {
8156 assert(
I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8157 "Expected G_CONSTANT");
8158 uint64_t CstVal =
I.getOperand(1).getCImm()->getZExtValue();
8163void AArch64InstructionSelector::renderLogicalImm64(
8164 MachineInstrBuilder &MIB,
const MachineInstr &
I,
int OpIdx)
const {
8165 assert(
I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
8166 "Expected G_CONSTANT");
8167 uint64_t CstVal =
I.getOperand(1).getCImm()->getZExtValue();
8172void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
8173 const MachineInstr &
MI,
8175 assert(
MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
8176 "Expected G_UBSANTRAP");
8177 MIB.
addImm(
MI.getOperand(0).getImm() | (
'U' << 8));
8180void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
8181 const MachineInstr &
MI,
8183 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8184 "Expected G_FCONSTANT");
8189void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
8190 const MachineInstr &
MI,
8192 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8193 "Expected G_FCONSTANT");
8198void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
8199 const MachineInstr &
MI,
8201 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
8202 "Expected G_FCONSTANT");
8207bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
8208 const MachineInstr &
MI,
unsigned NumBytes)
const {
8209 if (!
MI.mayLoadOrStore())
8212 "Expected load/store to have only one mem op!");
8213 return (*
MI.memoperands_begin())->getSize() == NumBytes;
8216bool AArch64InstructionSelector::isDef32(
const MachineInstr &
MI)
const {
8217 const MachineRegisterInfo &
MRI =
MI.getParent()->getParent()->getRegInfo();
8225 switch (
MI.getOpcode()) {
8228 case TargetOpcode::COPY:
8229 case TargetOpcode::G_BITCAST:
8230 case TargetOpcode::G_TRUNC:
8231 case TargetOpcode::G_PHI:
8241 assert(
MI.getOpcode() == TargetOpcode::G_PHI &&
"Expected a G_PHI");
8244 assert(DstRB &&
"Expected PHI dst to have regbank assigned");
8255 auto *OpDef =
MRI.getVRegDef(OpReg);
8256 const LLT &Ty =
MRI.getType(OpReg);
8262 if (InsertPt != OpDefBB.
end() && InsertPt->isPHI())
8266 MRI.setRegBank(Copy.getReg(0), *DstRB);
8267 MO.setReg(Copy.getReg(0));
8276 for (
auto &BB : MF) {
8277 for (
auto &
MI : BB) {
8278 if (
MI.getOpcode() == TargetOpcode::G_PHI)
8283 for (
auto *
MI : Phis) {
8305 bool HasGPROp =
false, HasFPROp =
false;
8319 if (RB->
getID() == AArch64::GPRRegBankID)
8325 if (HasGPROp && HasFPROp)
8331InstructionSelector *
8335 return new AArch64InstructionSelector(TM,
Subtarget, RBI);
MachineInstrBuilder MachineInstrBuilder & DefMI
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static bool isPreferredADD(int64_t ImmOff)
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A))...
static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static void changeFPCCToANDAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Convert a DAG fp condition code to an AArch64 CC.
const AArch64Subtarget & Subtarget
static bool canEmitConjunction(SelectionDAG &DAG, const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool &PreferFirst, bool WillNegate, unsigned Depth=0)
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
const TargetLowering & TLI
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert, MachineRegisterInfo &MRI)
Return a register which can be used as a bit to test in a TB(N)Z.
static unsigned getMinSizeForRegBank(const RegisterBank &RB)
Returns the minimum size the given register bank can hold.
static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the G_LOAD or G_STORE operation GenericOpc, appropriate for the (value)...
static const TargetRegisterClass * getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits, bool GetAllRegSet=false)
Given a register bank, and size in bits, return the smallest register class that can represent that c...
static std::optional< APInt > getVectorShiftImm(Register Reg, MachineRegisterInfo &MRI)
Returns the element immediate value of a vector shift operand if found.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc, appropriate for the register ban...
static bool getSubRegForClass(const TargetRegisterClass *RC, const TargetRegisterInfo &TRI, unsigned &SubReg)
Returns the correct subregister to use for a given register class.
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI, const RegisterBankInfo &RBI, Register SrcReg, const TargetRegisterClass *To, unsigned SubReg)
Helper function for selectCopy.
static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS={}, MachineRegisterInfo *MRI=nullptr)
static std::optional< uint64_t > getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI)
Matches and returns the shift immediate value for a SHL instruction given a shift operand.
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI, const AArch64RegisterBankInfo &RBI)
static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI)
Given a shift instruction, return the correct shift type for that instruction.
static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg, const unsigned EltSize)
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static std::optional< uint64_t > getImmedFromMO(const MachineOperand &Root)
static std::pair< unsigned, unsigned > getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize)
Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given size and RB.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
static void changeFPCCToORAArch64CC(CmpInst::Predicate CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file declares the targeting of the RegisterBankInfo class for AArch64.
static bool isStore(int Opcode)
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file contains constants used for implementing Dwarf debug support.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static MachineBasicBlock * emitSelect(MachineInstr &MI, MachineBasicBlock *BB, const TargetInstrInfo *TII, const PPCSubtarget &Subtarget)
Emit SELECT instruction, using ISEL if available, otherwise use branch-based control flow.
static StringRef getName(Value *V)
MachineRegisterInfo & MRI
static constexpr int Concat[]
unsigned getVarArgsFPRSize() const
bool hasELFSignedGOT() const
int getVarArgsFPRIndex() const
void incNumLocalDynamicTLSAccesses()
int getVarArgsStackIndex() const
int getVarArgsGPRIndex() const
unsigned getVarArgsGPRSize() const
This class provides the information for the target register banks.
bool isTargetDarwin() const
bool isTargetILP32() const
std::optional< uint16_t > getPtrAuthBlockAddressDiscriminatorIfEnabled(const Function &ParentFn) const
Compute the integer discriminator for a given BlockAddress constant, if blockaddress signing is enabl...
const AArch64TargetLowering * getTargetLowering() const override
bool isTargetMachO() const
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
bool isLittleEndian() const
bool isX16X17Safer() const
Returns whether the operating system makes it safer to store sensitive values in x16 and x17 as oppos...
bool isCallingConvWin64(CallingConv::ID CC, bool IsVarArg) const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
unsigned countr_one() const
Count the number of trailing one bits.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
bool isEquality() const
Determine if this is an equals/not equals predicate.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
bool isIntPredicate() const
static LLVM_ABI Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
const APFloat & getValue() const
const APFloat & getValueAPF() const
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
virtual void setupMF(MachineFunction &mf, GISelValueTracking *vt, CodeGenCoverage *covinfo=nullptr, ProfileSummaryInfo *psi=nullptr, BlockFrequencyInfo *bfi=nullptr)
Setup per-MF executor state.
Represents indexed stores.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSize() const
Returns the size in bytes of the memory access.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
Register getCondReg() const
Register getFalseReg() const
Register getTrueReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
bool hasExternalWeakLinkage() const
bool isEquality() const
Return true if this predicate is either EQ or NE.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
LLT multiplyElements(int Factor) const
Produce a vector type that is Factor times bigger, preserving the element type.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
LLT getScalarType() const
constexpr bool isPointerVector() const
constexpr bool isInteger() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
static LLT integer(unsigned SizeInBits)
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
TypeSize getValue() const
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
LLVM_ABI unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
void setAdjustsStack(bool V)
void setFrameAddressIsTaken(bool T)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineIRBuilderState & getState()
Getter for the State.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
void setState(const MachineIRBuilderState &NewState)
Setter for the State.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
Register getReg(unsigned Idx) const
Get the register for the operand index.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addBlockAddress(const BlockAddress *BA, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool isDebugInstr() const
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
void setImplicitPhysRegDefsDead()
Mark the implicit physreg defs named by the instruction description as dead.
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
AtomicOrdering getSuccessOrdering() const
Return the atomic ordering requirements for this memory operation.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LLVM_ABI void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreatePredicate(unsigned Pred)
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
static MachineOperand CreateBA(const BlockAddress *BA, int64_t Offset, unsigned TargetFlags=0)
const ConstantFP * getFPImm() const
unsigned getPredicate() const
int64_t getOffset() const
Return the offset from the symbol in this operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
const RegClassOrRegBank & getRegClassOrRegBank(Register Reg) const
Return the register bank or register class of Reg.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
def_instr_iterator def_instr_begin(Register RegNo) const
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
const RegisterBank * getRegBankOrNull(Register Reg) const
Return the register bank of Reg, or null if Reg has not been assigned a register bank or has been ass...
LLVM_ABI void setRegBank(Register Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
const MachineFunction & getMF() const
bool hasOneDef(Register RegNo) const
Return true if there is exactly one operand defining the specified register.
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
LLVM_ABI Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
Analysis providing profile information.
Holds all the information related to register banks.
static const TargetRegisterClass * constrainGenericRegister(Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI)
Constrain the (possibly generic) virtual register Reg to RC.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
TargetInstrInfo - Interface to description of machine instruction set.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
bool isPositionIndependent() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned TLSSize
Bit size of immediate TLS offsets (0 == use the default).
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
virtual const TargetLowering * getTargetLowering() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
void changeFCMPPredToAArch64CC(const CmpInst::Predicate P, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Find the AArch64 condition codes necessary to represent P for a scalar floating point comparison.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type)
isSignExtendShiftType - Returns true if Type is sign extending.
bool expandVectorMOVImm(APInt Imm, SmallVectorImpl< ImmInsnModel > &Insn, const AArch64Subtarget &ST)
TLSModel::Model getELFTLSModel(const GlobalValue *GV, const AArch64TargetMachine &TM, bool HasELFSignedGOT)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
auto m_PosZeroFP()
Matches a floating-point positive zero.
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
NodeAddr< InstrNode * > Instr
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
PointerUnion< const TargetRegisterClass *, const RegisterBank * > RegClassOrRegBank
Convenient type to represent either a register class or a register bank.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
unsigned CheckFixedPointOperandConstant(APFloat &FVal, unsigned RegWidth, bool isReciprocal)
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool isStrongerThanMonotonic(AtomicOrdering AO)
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ O1
Optimize quickly without destroying debuggability.
@ O0
Disable as many optimizations as possible.
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
InstructionSelector * createAArch64InstructionSelector(const AArch64TargetMachine &, const AArch64Subtarget &, const AArch64RegisterBankInfo &)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
LLVM_ABI std::optional< ValueAndVReg > getAnyConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT or G_FCONST...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Sub
Subtraction of integers.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
LLVM_ABI std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
LLVM_ABI std::optional< Register > getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
MCRegisterClass TargetRegisterClass
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.