44#include "llvm/IR/IntrinsicsAArch64.h"
52#define DEBUG_TYPE "aarch64-isel"
55using namespace MIPatternMatch;
56using namespace AArch64GISelUtils;
65#define GET_GLOBALISEL_PREDICATE_BITSET
66#include "AArch64GenGlobalISel.inc"
67#undef GET_GLOBALISEL_PREDICATE_BITSET
82 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
87 ProduceNonFlagSettingCondBr =
135 bool tryOptAndIntoCompareBranch(
MachineInstr &AndInst,
bool Invert,
213 bool selectVectorLoadIntrinsic(
unsigned Opc,
unsigned NumVecs,
215 bool selectVectorLoadLaneIntrinsic(
unsigned Opc,
unsigned NumVecs,
217 void selectVectorStoreIntrinsic(
MachineInstr &
I,
unsigned NumVecs,
219 bool selectVectorStoreLaneIntrinsic(
MachineInstr &
I,
unsigned NumVecs,
233 unsigned Opc1,
unsigned Opc2,
bool isExt);
239 unsigned emitConstantPoolEntry(
const Constant *CPVal,
258 std::optional<CmpInst::Predicate> = std::nullopt)
const;
261 emitInstr(
unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
262 std::initializer_list<llvm::SrcOp> SrcOps,
264 const ComplexRendererFns &RenderFns = std::nullopt)
const;
299 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
320 MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
342 std::pair<MachineInstr *, AArch64CC::CondCode>
377 ComplexRendererFns selectShiftA_32(
const MachineOperand &Root)
const;
378 ComplexRendererFns selectShiftB_32(
const MachineOperand &Root)
const;
379 ComplexRendererFns selectShiftA_64(
const MachineOperand &Root)
const;
380 ComplexRendererFns selectShiftB_64(
const MachineOperand &Root)
const;
382 ComplexRendererFns select12BitValueWithLeftShift(
uint64_t Immed)
const;
384 ComplexRendererFns selectNegArithImmed(
MachineOperand &Root)
const;
387 unsigned Size)
const;
389 ComplexRendererFns selectAddrModeUnscaled8(
MachineOperand &Root)
const {
390 return selectAddrModeUnscaled(Root, 1);
392 ComplexRendererFns selectAddrModeUnscaled16(
MachineOperand &Root)
const {
393 return selectAddrModeUnscaled(Root, 2);
395 ComplexRendererFns selectAddrModeUnscaled32(
MachineOperand &Root)
const {
396 return selectAddrModeUnscaled(Root, 4);
398 ComplexRendererFns selectAddrModeUnscaled64(
MachineOperand &Root)
const {
399 return selectAddrModeUnscaled(Root, 8);
401 ComplexRendererFns selectAddrModeUnscaled128(
MachineOperand &Root)
const {
402 return selectAddrModeUnscaled(Root, 16);
407 ComplexRendererFns tryFoldAddLowIntoImm(
MachineInstr &RootDef,
unsigned Size,
411 unsigned Size)
const;
413 ComplexRendererFns selectAddrModeIndexed(
MachineOperand &Root)
const {
414 return selectAddrModeIndexed(Root, Width / 8);
421 unsigned SizeInBytes)
const;
429 bool WantsExt)
const;
430 ComplexRendererFns selectAddrModeRegisterOffset(
MachineOperand &Root)
const;
432 unsigned SizeInBytes)
const;
434 ComplexRendererFns selectAddrModeXRO(
MachineOperand &Root)
const {
435 return selectAddrModeXRO(Root, Width / 8);
439 unsigned SizeInBytes)
const;
441 ComplexRendererFns selectAddrModeWRO(
MachineOperand &Root)
const {
442 return selectAddrModeWRO(Root, Width / 8);
446 bool AllowROR =
false)
const;
448 ComplexRendererFns selectArithShiftedRegister(
MachineOperand &Root)
const {
449 return selectShiftedRegister(Root);
452 ComplexRendererFns selectLogicalShiftedRegister(
MachineOperand &Root)
const {
453 return selectShiftedRegister(Root,
true);
463 bool IsLoadStore =
false)
const;
474 ComplexRendererFns selectArithExtendedRegister(
MachineOperand &Root)
const;
479 int OpIdx = -1)
const;
481 int OpIdx = -1)
const;
483 int OpIdx = -1)
const;
487 int OpIdx = -1)
const;
489 int OpIdx = -1)
const;
491 int OpIdx = -1)
const;
494 int OpIdx = -1)
const;
500 bool tryOptSelect(
GSelect &Sel);
507 bool isLoadStoreOfNumBytes(
const MachineInstr &
MI,
unsigned NumBytes)
const;
520 bool ProduceNonFlagSettingCondBr =
false;
529#define GET_GLOBALISEL_PREDICATES_DECL
530#include "AArch64GenGlobalISel.inc"
531#undef GET_GLOBALISEL_PREDICATES_DECL
535#define GET_GLOBALISEL_TEMPORARIES_DECL
536#include "AArch64GenGlobalISel.inc"
537#undef GET_GLOBALISEL_TEMPORARIES_DECL
542#define GET_GLOBALISEL_IMPL
543#include "AArch64GenGlobalISel.inc"
544#undef GET_GLOBALISEL_IMPL
546AArch64InstructionSelector::AArch64InstructionSelector(
549 :
TM(
TM), STI(STI),
TII(*STI.getInstrInfo()),
TRI(*STI.getRegisterInfo()),
552#include
"AArch64GenGlobalISel.inc"
555#include
"AArch64GenGlobalISel.inc"
567 bool GetAllRegSet =
false) {
568 if (RB.
getID() == AArch64::GPRRegBankID) {
570 return GetAllRegSet ? &AArch64::GPR32allRegClass
571 : &AArch64::GPR32RegClass;
573 return GetAllRegSet ? &AArch64::GPR64allRegClass
574 : &AArch64::GPR64RegClass;
576 return &AArch64::XSeqPairsClassRegClass;
580 if (RB.
getID() == AArch64::FPRRegBankID) {
583 return &AArch64::FPR8RegClass;
585 return &AArch64::FPR16RegClass;
587 return &AArch64::FPR32RegClass;
589 return &AArch64::FPR64RegClass;
591 return &AArch64::FPR128RegClass;
603 bool GetAllRegSet =
false) {
606 "Expected FPR regbank for scalable type size");
607 return &AArch64::ZPRRegClass;
610 unsigned RegBankID = RB.
getID();
612 if (RegBankID == AArch64::GPRRegBankID) {
613 if (SizeInBits <= 32)
614 return GetAllRegSet ? &AArch64::GPR32allRegClass
615 : &AArch64::GPR32RegClass;
616 if (SizeInBits == 64)
617 return GetAllRegSet ? &AArch64::GPR64allRegClass
618 : &AArch64::GPR64RegClass;
619 if (SizeInBits == 128)
620 return &AArch64::XSeqPairsClassRegClass;
623 if (RegBankID == AArch64::FPRRegBankID) {
624 switch (SizeInBits) {
628 return &AArch64::FPR8RegClass;
630 return &AArch64::FPR16RegClass;
632 return &AArch64::FPR32RegClass;
634 return &AArch64::FPR64RegClass;
636 return &AArch64::FPR128RegClass;
646 switch (
TRI.getRegSizeInBits(*RC)) {
654 if (RC != &AArch64::FPR32RegClass)
664 dbgs() <<
"Couldn't find appropriate subregister for register class.");
673 switch (RB.
getID()) {
674 case AArch64::GPRRegBankID:
676 case AArch64::FPRRegBankID:
699 const unsigned RegClassIDs[],
701 unsigned NumRegs = Regs.
size();
704 assert(NumRegs >= 2 && NumRegs <= 4 &&
705 "Only support between two and 4 registers in a tuple!");
707 auto *DesiredClass =
TRI->getRegClass(RegClassIDs[NumRegs - 2]);
709 MIB.
buildInstr(TargetOpcode::REG_SEQUENCE, {DesiredClass}, {});
710 for (
unsigned I = 0, E = Regs.
size();
I < E; ++
I) {
711 RegSequence.addUse(Regs[
I]);
712 RegSequence.addImm(SubRegs[
I]);
714 return RegSequence.getReg(0);
719 static const unsigned RegClassIDs[] = {
720 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
721 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
722 AArch64::dsub2, AArch64::dsub3};
723 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
728 static const unsigned RegClassIDs[] = {
729 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
730 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
731 AArch64::qsub2, AArch64::qsub3};
732 return createTuple(Regs, RegClassIDs, SubRegs, MIB);
737 auto &
MBB = *
MI.getParent();
739 auto &
MRI = MF.getRegInfo();
745 else if (Root.
isReg()) {
750 Immed = ValAndVReg->Value.getSExtValue();
766 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
773 for (
auto &MO :
I.operands()) {
776 LLVM_DEBUG(
dbgs() <<
"Generic inst non-reg operands are unsupported\n");
784 if (!MO.getReg().isVirtual()) {
785 LLVM_DEBUG(
dbgs() <<
"Generic inst has physical register operand\n");
795 if (PrevOpBank && OpBank != PrevOpBank) {
796 LLVM_DEBUG(
dbgs() <<
"Generic inst operands have different banks\n");
811 case AArch64::GPRRegBankID:
813 switch (GenericOpc) {
814 case TargetOpcode::G_SHL:
815 return AArch64::LSLVWr;
816 case TargetOpcode::G_LSHR:
817 return AArch64::LSRVWr;
818 case TargetOpcode::G_ASHR:
819 return AArch64::ASRVWr;
823 }
else if (OpSize == 64) {
824 switch (GenericOpc) {
825 case TargetOpcode::G_PTR_ADD:
826 return AArch64::ADDXrr;
827 case TargetOpcode::G_SHL:
828 return AArch64::LSLVXr;
829 case TargetOpcode::G_LSHR:
830 return AArch64::LSRVXr;
831 case TargetOpcode::G_ASHR:
832 return AArch64::ASRVXr;
838 case AArch64::FPRRegBankID:
841 switch (GenericOpc) {
842 case TargetOpcode::G_FADD:
843 return AArch64::FADDSrr;
844 case TargetOpcode::G_FSUB:
845 return AArch64::FSUBSrr;
846 case TargetOpcode::G_FMUL:
847 return AArch64::FMULSrr;
848 case TargetOpcode::G_FDIV:
849 return AArch64::FDIVSrr;
854 switch (GenericOpc) {
855 case TargetOpcode::G_FADD:
856 return AArch64::FADDDrr;
857 case TargetOpcode::G_FSUB:
858 return AArch64::FSUBDrr;
859 case TargetOpcode::G_FMUL:
860 return AArch64::FMULDrr;
861 case TargetOpcode::G_FDIV:
862 return AArch64::FDIVDrr;
863 case TargetOpcode::G_OR:
864 return AArch64::ORRv8i8;
881 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
883 case AArch64::GPRRegBankID:
886 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
888 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
890 return isStore ? AArch64::STRWui : AArch64::LDRWui;
892 return isStore ? AArch64::STRXui : AArch64::LDRXui;
895 case AArch64::FPRRegBankID:
898 return isStore ? AArch64::STRBui : AArch64::LDRBui;
900 return isStore ? AArch64::STRHui : AArch64::LDRHui;
902 return isStore ? AArch64::STRSui : AArch64::LDRSui;
904 return isStore ? AArch64::STRDui : AArch64::LDRDui;
906 return isStore ? AArch64::STRQui : AArch64::LDRQui;
920 assert(SrcReg.
isValid() &&
"Expected a valid source register?");
921 assert(To &&
"Destination register class cannot be null");
928 RegOp.
setReg(SubRegCopy.getReg(0));
932 if (!
I.getOperand(0).getReg().isPhysical())
942static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
946 Register DstReg =
I.getOperand(0).getReg();
947 Register SrcReg =
I.getOperand(1).getReg();
962 if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
980 if (Reg.isPhysical())
982 LLT Ty =
MRI.getType(Reg);
988 RC = getRegClassForTypeOnBank(Ty, RB);
991 dbgs() <<
"Warning: DBG_VALUE operand has unexpected size/bank\n");
1004 Register DstReg =
I.getOperand(0).getReg();
1005 Register SrcReg =
I.getOperand(1).getReg();
1024 LLVM_DEBUG(
dbgs() <<
"Couldn't determine source register class\n");
1028 const TypeSize SrcSize =
TRI.getRegSizeInBits(*SrcRC);
1029 const TypeSize DstSize =
TRI.getRegSizeInBits(*DstRC);
1040 auto Copy = MIB.
buildCopy({DstTempRC}, {SrcReg});
1042 }
else if (SrcSize > DstSize) {
1049 }
else if (DstSize > SrcSize) {
1056 Register PromoteReg =
MRI.createVirtualRegister(PromotionRC);
1058 TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
1063 RegOp.
setReg(PromoteReg);
1082 if (
I.getOpcode() == TargetOpcode::G_ZEXT) {
1083 I.setDesc(
TII.get(AArch64::COPY));
1084 assert(SrcRegBank.
getID() == AArch64::GPRRegBankID);
1088 I.setDesc(
TII.get(AArch64::COPY));
1103 switch (GenericOpc) {
1104 case TargetOpcode::G_SITOFP:
1105 return AArch64::SCVTFUWSri;
1106 case TargetOpcode::G_UITOFP:
1107 return AArch64::UCVTFUWSri;
1108 case TargetOpcode::G_FPTOSI:
1109 return AArch64::FCVTZSUWSr;
1110 case TargetOpcode::G_FPTOUI:
1111 return AArch64::FCVTZUUWSr;
1116 switch (GenericOpc) {
1117 case TargetOpcode::G_SITOFP:
1118 return AArch64::SCVTFUXSri;
1119 case TargetOpcode::G_UITOFP:
1120 return AArch64::UCVTFUXSri;
1121 case TargetOpcode::G_FPTOSI:
1122 return AArch64::FCVTZSUWDr;
1123 case TargetOpcode::G_FPTOUI:
1124 return AArch64::FCVTZUUWDr;
1134 switch (GenericOpc) {
1135 case TargetOpcode::G_SITOFP:
1136 return AArch64::SCVTFUWDri;
1137 case TargetOpcode::G_UITOFP:
1138 return AArch64::UCVTFUWDri;
1139 case TargetOpcode::G_FPTOSI:
1140 return AArch64::FCVTZSUXSr;
1141 case TargetOpcode::G_FPTOUI:
1142 return AArch64::FCVTZUUXSr;
1147 switch (GenericOpc) {
1148 case TargetOpcode::G_SITOFP:
1149 return AArch64::SCVTFUXDri;
1150 case TargetOpcode::G_UITOFP:
1151 return AArch64::UCVTFUXDri;
1152 case TargetOpcode::G_FPTOSI:
1153 return AArch64::FCVTZSUXDr;
1154 case TargetOpcode::G_FPTOUI:
1155 return AArch64::FCVTZUUXDr;
1174 RBI.getRegBank(True,
MRI,
TRI)->getID() &&
1175 "Expected both select operands to have the same regbank?");
1176 LLT Ty =
MRI.getType(True);
1181 "Expected 32 bit or 64 bit select only?");
1182 const bool Is32Bit =
Size == 32;
1183 if (RBI.getRegBank(True,
MRI,
TRI)->getID() != AArch64::GPRRegBankID) {
1184 unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
1185 auto FCSel = MIB.
buildInstr(Opc, {Dst}, {True, False}).addImm(
CC);
1191 unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
1193 auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &
CC, &
MRI,
1208 Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
1225 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1244 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1260 auto TryOptSelectCst = [&Opc, &True, &False, &
CC, Is32Bit, &
MRI,
1266 if (!TrueCst && !FalseCst)
1269 Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
1270 if (TrueCst && FalseCst) {
1271 int64_t
T = TrueCst->Value.getSExtValue();
1272 int64_t
F = FalseCst->Value.getSExtValue();
1274 if (
T == 0 &&
F == 1) {
1276 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1282 if (
T == 0 &&
F == -1) {
1284 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1292 int64_t
T = TrueCst->Value.getSExtValue();
1295 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1304 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1313 int64_t
F = FalseCst->Value.getSExtValue();
1316 Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
1323 Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
1331 Optimized |= TryFoldBinOpIntoSelect(False, True,
false);
1332 Optimized |= TryFoldBinOpIntoSelect(True, False,
true);
1453 assert(Reg.isValid() &&
"Expected valid register!");
1454 bool HasZext =
false;
1456 unsigned Opc =
MI->getOpcode();
1458 if (!
MI->getOperand(0).isReg() ||
1459 !
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
1466 if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
1467 Opc == TargetOpcode::G_TRUNC) {
1468 if (Opc == TargetOpcode::G_ZEXT)
1471 Register NextReg =
MI->getOperand(1).getReg();
1473 if (!NextReg.
isValid() || !
MRI.hasOneNonDBGUse(NextReg))
1482 std::optional<uint64_t>
C;
1487 case TargetOpcode::G_AND:
1488 case TargetOpcode::G_XOR: {
1489 TestReg =
MI->getOperand(1).getReg();
1490 Register ConstantReg =
MI->getOperand(2).getReg();
1501 C = VRegAndVal->Value.getZExtValue();
1503 C = VRegAndVal->Value.getSExtValue();
1507 case TargetOpcode::G_ASHR:
1508 case TargetOpcode::G_LSHR:
1509 case TargetOpcode::G_SHL: {
1510 TestReg =
MI->getOperand(1).getReg();
1514 C = VRegAndVal->Value.getSExtValue();
1526 unsigned TestRegSize =
MRI.getType(TestReg).getSizeInBits();
1530 case TargetOpcode::G_AND:
1532 if ((*
C >> Bit) & 1)
1535 case TargetOpcode::G_SHL:
1538 if (*
C <= Bit && (Bit - *
C) < TestRegSize) {
1543 case TargetOpcode::G_ASHR:
1548 if (Bit >= TestRegSize)
1549 Bit = TestRegSize - 1;
1551 case TargetOpcode::G_LSHR:
1553 if ((Bit + *
C) < TestRegSize) {
1558 case TargetOpcode::G_XOR:
1567 if ((*
C >> Bit) & 1)
1586 assert(ProduceNonFlagSettingCondBr &&
1587 "Cannot emit TB(N)Z with speculation tracking!");
1592 LLT Ty =
MRI.getType(TestReg);
1595 assert(Bit < 64 &&
"Bit is too large!");
1599 bool UseWReg =
Bit < 32;
1600 unsigned NecessarySize = UseWReg ? 32 : 64;
1601 if (
Size != NecessarySize)
1602 TestReg = moveScalarRegClass(
1603 TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
1606 static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
1607 {AArch64::TBZW, AArch64::TBNZW}};
1608 unsigned Opc = OpcTable[UseWReg][IsNegative];
1615bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
1618 assert(AndInst.
getOpcode() == TargetOpcode::G_AND &&
"Expected G_AND only?");
1645 int32_t
Bit = MaybeBit->Value.exactLogBase2();
1652 emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
1660 assert(ProduceNonFlagSettingCondBr &&
"CBZ does not set flags!");
1662 assert(RBI.getRegBank(CompareReg,
MRI,
TRI)->getID() ==
1663 AArch64::GPRRegBankID &&
1664 "Expected GPRs only?");
1665 auto Ty =
MRI.getType(CompareReg);
1668 assert(Width <= 64 &&
"Expected width to be at most 64?");
1669 static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
1670 {AArch64::CBNZW, AArch64::CBNZX}};
1671 unsigned Opc = OpcTable[IsNegative][Width == 64];
1672 auto BranchMI = MIB.
buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
1677bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
1680 assert(
I.getOpcode() == TargetOpcode::G_BRCOND);
1692 I.eraseFromParent();
1696bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
1699 assert(
I.getOpcode() == TargetOpcode::G_BRCOND);
1705 if (!ProduceNonFlagSettingCondBr)
1724 if (VRegAndVal && !AndInst) {
1725 int64_t
C = VRegAndVal->Value.getSExtValue();
1731 emitTestBit(LHS, Bit,
false, DestMBB, MIB);
1732 I.eraseFromParent();
1740 emitTestBit(LHS, Bit,
true, DestMBB, MIB);
1741 I.eraseFromParent();
1749 emitTestBit(LHS, Bit,
false, DestMBB, MIB);
1750 I.eraseFromParent();
1764 if (VRegAndVal && VRegAndVal->Value == 0) {
1772 tryOptAndIntoCompareBranch(
1774 I.eraseFromParent();
1779 auto LHSTy =
MRI.getType(LHS);
1780 if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
1782 I.eraseFromParent();
1791bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
1794 assert(
I.getOpcode() == TargetOpcode::G_BRCOND);
1795 if (tryOptCompareBranchFedByICmp(
I, ICmp, MIB))
1805 I.eraseFromParent();
1809bool AArch64InstructionSelector::selectCompareBranch(
1811 Register CondReg =
I.getOperand(0).getReg();
1816 if (CCMIOpc == TargetOpcode::G_FCMP)
1817 return selectCompareBranchFedByFCmp(
I, *CCMI, MIB);
1818 if (CCMIOpc == TargetOpcode::G_ICMP)
1819 return selectCompareBranchFedByICmp(
I, *CCMI, MIB);
1824 if (ProduceNonFlagSettingCondBr) {
1825 emitTestBit(CondReg, 0,
true,
1826 I.getOperand(1).getMBB(), MIB);
1827 I.eraseFromParent();
1837 .
addMBB(
I.getOperand(1).getMBB());
1838 I.eraseFromParent();
1846 assert(
MRI.getType(Reg).isVector() &&
"Expected a *vector* shift operand");
1857 return std::nullopt;
1859 int64_t Imm = *ShiftImm;
1861 return std::nullopt;
1865 return std::nullopt;
1868 return std::nullopt;
1872 return std::nullopt;
1876 return std::nullopt;
1880 return std::nullopt;
1886bool AArch64InstructionSelector::selectVectorSHL(
MachineInstr &
I,
1888 assert(
I.getOpcode() == TargetOpcode::G_SHL);
1889 Register DstReg =
I.getOperand(0).getReg();
1890 const LLT Ty =
MRI.getType(DstReg);
1891 Register Src1Reg =
I.getOperand(1).getReg();
1892 Register Src2Reg =
I.getOperand(2).getReg();
1903 Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
1905 Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
1907 Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
1909 Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
1911 Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
1913 Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
1915 Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
1921 auto Shl = MIB.
buildInstr(Opc, {DstReg}, {Src1Reg});
1927 I.eraseFromParent();
1931bool AArch64InstructionSelector::selectVectorAshrLshr(
1933 assert(
I.getOpcode() == TargetOpcode::G_ASHR ||
1934 I.getOpcode() == TargetOpcode::G_LSHR);
1935 Register DstReg =
I.getOperand(0).getReg();
1936 const LLT Ty =
MRI.getType(DstReg);
1937 Register Src1Reg =
I.getOperand(1).getReg();
1938 Register Src2Reg =
I.getOperand(2).getReg();
1943 bool IsASHR =
I.getOpcode() == TargetOpcode::G_ASHR;
1953 unsigned NegOpc = 0;
1955 getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID));
1957 Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
1958 NegOpc = AArch64::NEGv2i64;
1960 Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
1961 NegOpc = AArch64::NEGv4i32;
1963 Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
1964 NegOpc = AArch64::NEGv2i32;
1966 Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
1967 NegOpc = AArch64::NEGv4i16;
1969 Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
1970 NegOpc = AArch64::NEGv8i16;
1972 Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
1973 NegOpc = AArch64::NEGv16i8;
1975 Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
1976 NegOpc = AArch64::NEGv8i8;
1982 auto Neg = MIB.
buildInstr(NegOpc, {RC}, {Src2Reg});
1984 auto SShl = MIB.
buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
1986 I.eraseFromParent();
1990bool AArch64InstructionSelector::selectVaStartAAPCS(
1995bool AArch64InstructionSelector::selectVaStartDarwin(
1998 Register ListReg =
I.getOperand(0).getReg();
2000 Register ArgsAddrReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
2011 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::ADDXri))
2019 MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::STRXui))
2026 I.eraseFromParent();
2030void AArch64InstructionSelector::materializeLargeCMVal(
2036 auto MovZ = MIB.
buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
2047 :
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
2049 if (
auto *GV = dyn_cast<GlobalValue>(V)) {
2051 GV, MovZ->getOperand(1).getOffset(), Flags));
2055 MovZ->getOperand(1).getOffset(), Flags));
2061 Register DstReg = BuildMovK(MovZ.getReg(0),
2067bool AArch64InstructionSelector::preISelLower(
MachineInstr &
I) {
2072 switch (
I.getOpcode()) {
2073 case TargetOpcode::G_STORE: {
2074 bool Changed = contractCrossBankCopyIntoStore(
I,
MRI);
2082 SrcOp.setReg(NewSrc);
2083 RBI.constrainGenericRegister(NewSrc, AArch64::GPR64RegClass,
MRI);
2088 case TargetOpcode::G_PTR_ADD:
2089 return convertPtrAddToAdd(
I,
MRI);
2090 case TargetOpcode::G_LOAD: {
2095 Register DstReg =
I.getOperand(0).getReg();
2096 const LLT DstTy =
MRI.getType(DstReg);
2102 case AArch64::G_DUP: {
2104 LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
2108 MRI.setType(
I.getOperand(0).getReg(),
2110 MRI.setRegClass(NewSrc.getReg(0), &AArch64::GPR64RegClass);
2111 I.getOperand(1).setReg(NewSrc.getReg(0));
2114 case TargetOpcode::G_UITOFP:
2115 case TargetOpcode::G_SITOFP: {
2120 Register SrcReg =
I.getOperand(1).getReg();
2121 LLT SrcTy =
MRI.getType(SrcReg);
2122 LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
2126 if (RBI.getRegBank(SrcReg,
MRI,
TRI)->getID() == AArch64::FPRRegBankID) {
2127 if (
I.getOpcode() == TargetOpcode::G_SITOFP)
2128 I.setDesc(
TII.get(AArch64::G_SITOF));
2130 I.setDesc(
TII.get(AArch64::G_UITOF));
2148bool AArch64InstructionSelector::convertPtrAddToAdd(
2150 assert(
I.getOpcode() == TargetOpcode::G_PTR_ADD &&
"Expected G_PTR_ADD");
2151 Register DstReg =
I.getOperand(0).getReg();
2152 Register AddOp1Reg =
I.getOperand(1).getReg();
2153 const LLT PtrTy =
MRI.getType(DstReg);
2157 const LLT CastPtrTy =
2162 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
2164 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
2168 I.setDesc(
TII.get(TargetOpcode::G_ADD));
2169 MRI.setType(DstReg, CastPtrTy);
2170 I.getOperand(1).setReg(PtrToInt.getReg(0));
2171 if (!select(*PtrToInt)) {
2172 LLVM_DEBUG(
dbgs() <<
"Failed to select G_PTRTOINT in convertPtrAddToAdd");
2181 I.getOperand(2).setReg(NegatedReg);
2182 I.setDesc(
TII.get(TargetOpcode::G_SUB));
2186bool AArch64InstructionSelector::earlySelectSHL(
MachineInstr &
I,
2191 assert(
I.getOpcode() == TargetOpcode::G_SHL &&
"unexpected op");
2192 const auto &MO =
I.getOperand(2);
2197 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
2201 auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
2202 auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);
2204 if (!Imm1Fn || !Imm2Fn)
2208 MIB.
buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
2211 for (
auto &RenderFn : *Imm1Fn)
2213 for (
auto &RenderFn : *Imm2Fn)
2216 I.eraseFromParent();
2220bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
2222 assert(
I.getOpcode() == TargetOpcode::G_STORE &&
"Expected G_STORE");
2240 LLT DefDstTy =
MRI.getType(DefDstReg);
2241 Register StoreSrcReg =
I.getOperand(0).getReg();
2242 LLT StoreSrcTy =
MRI.getType(StoreSrcReg);
2253 if (RBI.getRegBank(StoreSrcReg,
MRI,
TRI) ==
2254 RBI.getRegBank(DefDstReg,
MRI,
TRI))
2258 I.getOperand(0).setReg(DefDstReg);
2262bool AArch64InstructionSelector::earlySelect(
MachineInstr &
I) {
2263 assert(
I.getParent() &&
"Instruction should be in a basic block!");
2264 assert(
I.getParent()->getParent() &&
"Instruction should be in a function!");
2270 switch (
I.getOpcode()) {
2271 case AArch64::G_DUP: {
2274 Register Src =
I.getOperand(1).getReg();
2279 Register Dst =
I.getOperand(0).getReg();
2281 MRI.getType(Dst).getNumElements(),
2283 ValAndVReg->Value));
2284 if (!emitConstantVector(Dst, CV, MIB,
MRI))
2286 I.eraseFromParent();
2289 case TargetOpcode::G_SEXT:
2292 if (selectUSMovFromExtend(
I,
MRI))
2295 case TargetOpcode::G_BR:
2297 case TargetOpcode::G_SHL:
2298 return earlySelectSHL(
I,
MRI);
2299 case TargetOpcode::G_CONSTANT: {
2300 bool IsZero =
false;
2301 if (
I.getOperand(1).isCImm())
2302 IsZero =
I.getOperand(1).getCImm()->isZero();
2303 else if (
I.getOperand(1).isImm())
2304 IsZero =
I.getOperand(1).getImm() == 0;
2309 Register DefReg =
I.getOperand(0).getReg();
2310 LLT Ty =
MRI.getType(DefReg);
2312 I.getOperand(1).ChangeToRegister(AArch64::XZR,
false);
2313 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
MRI);
2315 I.getOperand(1).ChangeToRegister(AArch64::WZR,
false);
2316 RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass,
MRI);
2320 I.setDesc(
TII.get(TargetOpcode::COPY));
2324 case TargetOpcode::G_ADD: {
2333 Register AddDst =
I.getOperand(0).getReg();
2334 Register AddLHS =
I.getOperand(1).getReg();
2335 Register AddRHS =
I.getOperand(2).getReg();
2337 LLT Ty =
MRI.getType(AddLHS);
2346 if (!
MRI.hasOneNonDBGUse(Reg))
2360 MRI.getType(
Cmp->getOperand(2).getReg()).getSizeInBits() != 64)
2370 Cmp = MatchCmp(AddRHS);
2374 auto &PredOp =
Cmp->getOperand(1);
2379 emitIntegerCompare(
Cmp->getOperand(2),
2380 Cmp->getOperand(3), PredOp, MIB);
2381 emitCSINC(AddDst, AddLHS, AddLHS, InvCC, MIB);
2382 I.eraseFromParent();
2385 case TargetOpcode::G_OR: {
2389 Register Dst =
I.getOperand(0).getReg();
2390 LLT Ty =
MRI.getType(Dst);
2409 if (ShiftImm >
Size || ((1ULL << ShiftImm) - 1ULL) !=
uint64_t(MaskImm))
2412 int64_t Immr =
Size - ShiftImm;
2413 int64_t Imms =
Size - ShiftImm - 1;
2414 unsigned Opc =
Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
2415 emitInstr(Opc, {Dst}, {MaskSrc, ShiftSrc, Immr, Imms}, MIB);
2416 I.eraseFromParent();
2419 case TargetOpcode::G_FENCE: {
2420 if (
I.getOperand(1).getImm() == 0)
2424 .
addImm(
I.getOperand(0).getImm() == 4 ? 0x9 : 0xb);
2425 I.eraseFromParent();
2434 assert(
I.getParent() &&
"Instruction should be in a basic block!");
2435 assert(
I.getParent()->getParent() &&
"Instruction should be in a function!");
2442 if (Subtarget->requiresStrictAlign()) {
2444 LLVM_DEBUG(
dbgs() <<
"AArch64 GISel does not support strict-align yet\n");
2450 unsigned Opcode =
I.getOpcode();
2452 if (!
I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
2455 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
2458 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
2459 const Register DefReg =
I.getOperand(0).getReg();
2460 const LLT DefTy =
MRI.getType(DefReg);
2463 MRI.getRegClassOrRegBank(DefReg);
2473 DefRC = getRegClassForTypeOnBank(DefTy, RB);
2480 I.setDesc(
TII.get(TargetOpcode::PHI));
2482 return RBI.constrainGenericRegister(DefReg, *DefRC,
MRI);
2488 if (
I.isDebugInstr())
2495 if (
I.getNumOperands() !=
I.getNumExplicitOperands()) {
2497 dbgs() <<
"Generic instruction has unexpected implicit operands\n");
2504 if (preISelLower(
I)) {
2505 Opcode =
I.getOpcode();
2516 if (selectImpl(
I, *CoverageInfo))
2520 I.getOperand(0).isReg() ?
MRI.getType(
I.getOperand(0).getReg()) :
LLT{};
2523 case TargetOpcode::G_SBFX:
2524 case TargetOpcode::G_UBFX: {
2525 static const unsigned OpcTable[2][2] = {
2526 {AArch64::UBFMWri, AArch64::UBFMXri},
2527 {AArch64::SBFMWri, AArch64::SBFMXri}};
2528 bool IsSigned = Opcode == TargetOpcode::G_SBFX;
2530 unsigned Opc = OpcTable[IsSigned][
Size == 64];
2533 assert(Cst1 &&
"Should have gotten a constant for src 1?");
2536 assert(Cst2 &&
"Should have gotten a constant for src 2?");
2537 auto LSB = Cst1->Value.getZExtValue();
2538 auto Width = Cst2->Value.getZExtValue();
2540 MIB.
buildInstr(Opc, {
I.getOperand(0)}, {
I.getOperand(1)})
2542 .
addImm(LSB + Width - 1);
2543 I.eraseFromParent();
2546 case TargetOpcode::G_BRCOND:
2547 return selectCompareBranch(
I, MF,
MRI);
2549 case TargetOpcode::G_BRINDIRECT: {
2550 I.setDesc(
TII.get(AArch64::BR));
2554 case TargetOpcode::G_BRJT:
2555 return selectBrJT(
I,
MRI);
2557 case AArch64::G_ADD_LOW: {
2563 if (BaseMI->
getOpcode() != AArch64::ADRP) {
2564 I.setDesc(
TII.get(AArch64::ADDXri));
2569 "Expected small code model");
2571 auto Op2 =
I.getOperand(2);
2572 auto MovAddr = MIB.
buildInstr(AArch64::MOVaddr, {
I.getOperand(0)}, {})
2573 .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
2574 Op1.getTargetFlags())
2576 Op2.getTargetFlags());
2577 I.eraseFromParent();
2581 case TargetOpcode::G_FCONSTANT:
2582 case TargetOpcode::G_CONSTANT: {
2583 const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
2592 const Register DefReg =
I.getOperand(0).getReg();
2593 const LLT DefTy =
MRI.getType(DefReg);
2599 if (Ty != s16 && Ty != s32 && Ty != s64 && Ty != s128) {
2601 <<
" constant, expected: " << s16 <<
" or " << s32
2602 <<
" or " << s64 <<
" or " << s128 <<
'\n');
2606 if (RB.
getID() != AArch64::FPRRegBankID) {
2608 <<
" constant on bank: " << RB
2609 <<
", expected: FPR\n");
2617 if (DefSize != 128 &&
I.getOperand(1).getFPImm()->isExactlyValue(0.0))
2621 if (Ty != p0 && Ty != s8 && Ty != s16) {
2623 <<
" constant, expected: " << s32 <<
", " << s64
2624 <<
", or " << p0 <<
'\n');
2628 if (RB.
getID() != AArch64::GPRRegBankID) {
2630 <<
" constant on bank: " << RB
2631 <<
", expected: GPR\n");
2648 if (TLI->isFPImmLegal(
I.getOperand(1).getFPImm()->getValueAPF(),
2655 auto *FPImm =
I.getOperand(1).getFPImm();
2658 LLVM_DEBUG(
dbgs() <<
"Failed to load double constant pool entry\n");
2662 I.eraseFromParent();
2663 return RBI.constrainGenericRegister(DefReg, FPRRC,
MRI);
2667 assert((DefSize == 32 || DefSize == 64) &&
"Unexpected const def size");
2669 const Register DefGPRReg =
MRI.createVirtualRegister(
2670 DefSize == 32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2676 if (!RBI.constrainGenericRegister(DefReg, FPRRC,
MRI)) {
2677 LLVM_DEBUG(
dbgs() <<
"Failed to constrain G_FCONSTANT def operand\n");
2685 }
else if (
I.getOperand(1).isCImm()) {
2686 uint64_t Val =
I.getOperand(1).getCImm()->getZExtValue();
2687 I.getOperand(1).ChangeToImmediate(Val);
2688 }
else if (
I.getOperand(1).isImm()) {
2689 uint64_t Val =
I.getOperand(1).getImm();
2690 I.getOperand(1).ChangeToImmediate(Val);
2693 const unsigned MovOpc =
2694 DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
2695 I.setDesc(
TII.get(MovOpc));
2699 case TargetOpcode::G_EXTRACT: {
2700 Register DstReg =
I.getOperand(0).getReg();
2701 Register SrcReg =
I.getOperand(1).getReg();
2702 LLT SrcTy =
MRI.getType(SrcReg);
2703 LLT DstTy =
MRI.getType(DstReg);
2715 unsigned Offset =
I.getOperand(2).getImm();
2724 if (SrcRB.
getID() == AArch64::GPRRegBankID) {
2726 MIB.
buildInstr(TargetOpcode::COPY, {DstReg}, {})
2728 Offset == 0 ? AArch64::sube64 : AArch64::subo64);
2730 AArch64::GPR64RegClass, NewI->getOperand(0));
2731 I.eraseFromParent();
2737 unsigned LaneIdx =
Offset / 64;
2739 DstReg, DstRB,
LLT::scalar(64), SrcReg, LaneIdx, MIB);
2742 I.eraseFromParent();
2746 I.setDesc(
TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
2752 "unexpected G_EXTRACT types");
2759 .addReg(DstReg, 0, AArch64::sub_32);
2760 RBI.constrainGenericRegister(
I.getOperand(0).getReg(),
2761 AArch64::GPR32RegClass,
MRI);
2762 I.getOperand(0).setReg(DstReg);
2767 case TargetOpcode::G_INSERT: {
2768 LLT SrcTy =
MRI.getType(
I.getOperand(2).getReg());
2769 LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
2776 I.setDesc(
TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
2777 unsigned LSB =
I.getOperand(3).getImm();
2778 unsigned Width =
MRI.getType(
I.getOperand(2).getReg()).getSizeInBits();
2779 I.getOperand(3).setImm((DstSize - LSB) % DstSize);
2784 "unexpected G_INSERT types");
2790 TII.get(AArch64::SUBREG_TO_REG))
2793 .
addUse(
I.getOperand(2).getReg())
2794 .
addImm(AArch64::sub_32);
2795 RBI.constrainGenericRegister(
I.getOperand(2).getReg(),
2796 AArch64::GPR32RegClass,
MRI);
2797 I.getOperand(2).setReg(SrcReg);
2801 case TargetOpcode::G_FRAME_INDEX: {
2808 I.setDesc(
TII.get(AArch64::ADDXri));
2817 case TargetOpcode::G_GLOBAL_VALUE: {
2820 if (
I.getOperand(1).isSymbol()) {
2821 OpFlags =
I.getOperand(1).getTargetFlags();
2825 GV =
I.getOperand(1).getGlobal();
2827 return selectTLSGlobalValue(
I,
MRI);
2828 OpFlags = STI.ClassifyGlobalReference(GV,
TM);
2832 I.setDesc(
TII.get(AArch64::LOADgot));
2833 I.getOperand(1).setTargetFlags(OpFlags);
2835 !
TM.isPositionIndependent()) {
2837 materializeLargeCMVal(
I, GV, OpFlags);
2838 I.eraseFromParent();
2841 I.setDesc(
TII.get(AArch64::ADR));
2842 I.getOperand(1).setTargetFlags(OpFlags);
2844 I.setDesc(
TII.get(AArch64::MOVaddr));
2847 MIB.addGlobalAddress(GV,
I.getOperand(1).getOffset(),
2853 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE:
2854 return selectPtrAuthGlobalValue(
I,
MRI);
2856 case TargetOpcode::G_ZEXTLOAD:
2857 case TargetOpcode::G_LOAD:
2858 case TargetOpcode::G_STORE: {
2860 bool IsZExtLoad =
I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
2874 if (Order != AtomicOrdering::NotAtomic &&
2875 Order != AtomicOrdering::Unordered &&
2876 Order != AtomicOrdering::Monotonic) {
2877 assert(!isa<GZExtLoad>(LdSt));
2878 assert(MemSizeInBytes <= 8 &&
2879 "128-bit atomics should already be custom-legalized");
2881 if (isa<GLoad>(LdSt)) {
2882 static constexpr unsigned LDAPROpcodes[] = {
2883 AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
2884 static constexpr unsigned LDAROpcodes[] = {
2885 AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
2887 STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
2890 I.setDesc(
TII.get(Opcodes[
Log2_32(MemSizeInBytes)]));
2892 static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
2893 AArch64::STLRW, AArch64::STLRX};
2895 if (
MRI.getType(ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
2897 Register NewVal =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
2898 MIB.
buildInstr(TargetOpcode::COPY, {NewVal}, {})
2899 .addReg(
I.getOperand(0).getReg(), 0, AArch64::sub_32);
2900 I.getOperand(0).setReg(NewVal);
2902 I.setDesc(
TII.get(Opcodes[
Log2_32(MemSizeInBytes)]));
2913 "Load/Store pointer operand isn't a GPR");
2914 assert(
MRI.getType(PtrReg).isPointer() &&
2915 "Load/Store pointer operand isn't a pointer");
2919 const LLT ValTy =
MRI.getType(ValReg);
2924 if (isa<GStore>(LdSt) && ValTy.
getSizeInBits() > MemSizeInBits) {
2927 auto *RC = getRegClassForTypeOnBank(MemTy, RB);
2933 .addReg(ValReg, 0,
SubReg)
2935 RBI.constrainGenericRegister(Copy, *RC,
MRI);
2937 }
else if (isa<GLoad>(LdSt) && ValTy.
getSizeInBits() > MemSizeInBits) {
2940 if (RB.
getID() == AArch64::FPRRegBankID) {
2943 auto *RC = getRegClassForTypeOnBank(MemTy, RB);
2950 MRI.setRegBank(NewDst, RB);
2953 MIB.
buildInstr(AArch64::SUBREG_TO_REG, {OldDst}, {})
2957 auto SubRegRC = getRegClassForTypeOnBank(
MRI.getType(OldDst), RB);
2958 RBI.constrainGenericRegister(OldDst, *SubRegRC,
MRI);
2965 auto SelectLoadStoreAddressingMode = [&]() ->
MachineInstr * {
2966 bool IsStore = isa<GStore>(
I);
2967 const unsigned NewOpc =
2969 if (NewOpc ==
I.getOpcode())
2973 selectAddrModeIndexed(
I.getOperand(1), MemSizeInBytes);
2976 I.setDesc(
TII.get(NewOpc));
2982 auto NewInst = MIB.
buildInstr(NewOpc, {}, {},
I.getFlags());
2983 Register CurValReg =
I.getOperand(0).getReg();
2984 IsStore ? NewInst.addUse(CurValReg) : NewInst.addDef(CurValReg);
2985 NewInst.cloneMemRefs(
I);
2986 for (
auto &Fn : *AddrModeFns)
2988 I.eraseFromParent();
2997 if (Opcode == TargetOpcode::G_STORE) {
3000 if (CVal && CVal->Value == 0) {
3002 case AArch64::STRWui:
3003 case AArch64::STRHHui:
3004 case AArch64::STRBBui:
3005 LoadStore->getOperand(0).setReg(AArch64::WZR);
3007 case AArch64::STRXui:
3008 LoadStore->getOperand(0).setReg(AArch64::XZR);
3014 if (IsZExtLoad || (Opcode == TargetOpcode::G_LOAD &&
3015 ValTy ==
LLT::scalar(64) && MemSizeInBits == 32)) {
3018 if (
MRI.getType(
LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
3022 Register LdReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3027 MIB.
buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
3030 .
addImm(AArch64::sub_32);
3032 return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
3038 case TargetOpcode::G_INDEXED_ZEXTLOAD:
3039 case TargetOpcode::G_INDEXED_SEXTLOAD:
3040 return selectIndexedExtLoad(
I,
MRI);
3041 case TargetOpcode::G_INDEXED_LOAD:
3042 return selectIndexedLoad(
I,
MRI);
3043 case TargetOpcode::G_INDEXED_STORE:
3044 return selectIndexedStore(cast<GIndexedStore>(
I),
MRI);
3046 case TargetOpcode::G_LSHR:
3047 case TargetOpcode::G_ASHR:
3048 if (
MRI.getType(
I.getOperand(0).getReg()).isVector())
3049 return selectVectorAshrLshr(
I,
MRI);
3051 case TargetOpcode::G_SHL:
3052 if (Opcode == TargetOpcode::G_SHL &&
3053 MRI.getType(
I.getOperand(0).getReg()).isVector())
3054 return selectVectorSHL(
I,
MRI);
3061 Register SrcReg =
I.getOperand(1).getReg();
3062 Register ShiftReg =
I.getOperand(2).getReg();
3063 const LLT ShiftTy =
MRI.getType(ShiftReg);
3064 const LLT SrcTy =
MRI.getType(SrcReg);
3069 auto Trunc = MIB.
buildInstr(TargetOpcode::COPY, {SrcTy}, {})
3070 .addReg(ShiftReg, 0, AArch64::sub_32);
3071 MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
3072 I.getOperand(2).setReg(Trunc.getReg(0));
3076 case TargetOpcode::G_OR: {
3083 const Register DefReg =
I.getOperand(0).getReg();
3087 if (NewOpc ==
I.getOpcode())
3090 I.setDesc(
TII.get(NewOpc));
3098 case TargetOpcode::G_PTR_ADD: {
3099 emitADD(
I.getOperand(0).getReg(),
I.getOperand(1),
I.getOperand(2), MIB);
3100 I.eraseFromParent();
3104 case TargetOpcode::G_SADDE:
3105 case TargetOpcode::G_UADDE:
3106 case TargetOpcode::G_SSUBE:
3107 case TargetOpcode::G_USUBE:
3108 case TargetOpcode::G_SADDO:
3109 case TargetOpcode::G_UADDO:
3110 case TargetOpcode::G_SSUBO:
3111 case TargetOpcode::G_USUBO:
3112 return selectOverflowOp(
I,
MRI);
3114 case TargetOpcode::G_PTRMASK: {
3115 Register MaskReg =
I.getOperand(2).getReg();
3122 I.setDesc(
TII.get(AArch64::ANDXri));
3123 I.getOperand(2).ChangeToImmediate(
3128 case TargetOpcode::G_PTRTOINT:
3129 case TargetOpcode::G_TRUNC: {
3130 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
3131 const LLT SrcTy =
MRI.getType(
I.getOperand(1).getReg());
3133 const Register DstReg =
I.getOperand(0).getReg();
3134 const Register SrcReg =
I.getOperand(1).getReg();
3141 dbgs() <<
"G_TRUNC/G_PTRTOINT input/output on different banks\n");
3145 if (DstRB.
getID() == AArch64::GPRRegBankID) {
3154 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC,
MRI) ||
3155 !RBI.constrainGenericRegister(DstReg, *DstRC,
MRI)) {
3156 LLVM_DEBUG(
dbgs() <<
"Failed to constrain G_TRUNC/G_PTRTOINT\n");
3160 if (DstRC == SrcRC) {
3162 }
else if (Opcode == TargetOpcode::G_TRUNC && DstTy ==
LLT::scalar(32) &&
3166 }
else if (DstRC == &AArch64::GPR32RegClass &&
3167 SrcRC == &AArch64::GPR64RegClass) {
3168 I.getOperand(1).setSubReg(AArch64::sub_32);
3171 dbgs() <<
"Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
3175 I.setDesc(
TII.get(TargetOpcode::COPY));
3177 }
else if (DstRB.
getID() == AArch64::FPRRegBankID) {
3180 I.setDesc(
TII.get(AArch64::XTNv4i16));
3190 I.eraseFromParent();
3195 if (Opcode == TargetOpcode::G_PTRTOINT) {
3196 assert(DstTy.
isVector() &&
"Expected an FPR ptrtoint to be a vector");
3197 I.setDesc(
TII.get(TargetOpcode::COPY));
3205 case TargetOpcode::G_ANYEXT: {
3206 if (selectUSMovFromExtend(
I,
MRI))
3209 const Register DstReg =
I.getOperand(0).getReg();
3210 const Register SrcReg =
I.getOperand(1).getReg();
3213 if (RBDst.
getID() != AArch64::GPRRegBankID) {
3215 <<
", expected: GPR\n");
3220 if (RBSrc.
getID() != AArch64::GPRRegBankID) {
3222 <<
", expected: GPR\n");
3226 const unsigned DstSize =
MRI.getType(DstReg).getSizeInBits();
3229 LLVM_DEBUG(
dbgs() <<
"G_ANYEXT operand has no size, not a gvreg?\n");
3233 if (DstSize != 64 && DstSize > 32) {
3235 <<
", expected: 32 or 64\n");
3241 Register ExtSrc =
MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
3246 .
addImm(AArch64::sub_32);
3247 I.getOperand(1).setReg(ExtSrc);
3252 case TargetOpcode::G_ZEXT:
3253 case TargetOpcode::G_SEXT_INREG:
3254 case TargetOpcode::G_SEXT: {
3255 if (selectUSMovFromExtend(
I,
MRI))
3258 unsigned Opcode =
I.getOpcode();
3259 const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
3260 const Register DefReg =
I.getOperand(0).getReg();
3261 Register SrcReg =
I.getOperand(1).getReg();
3262 const LLT DstTy =
MRI.getType(DefReg);
3263 const LLT SrcTy =
MRI.getType(SrcReg);
3269 if (Opcode == TargetOpcode::G_SEXT_INREG)
3270 SrcSize =
I.getOperand(2).getImm();
3276 AArch64::GPRRegBankID &&
3277 "Unexpected ext regbank");
3290 RBI.getRegBank(SrcReg,
MRI,
TRI)->getID() == AArch64::GPRRegBankID;
3291 if (LoadMI && IsGPR) {
3293 unsigned BytesLoaded =
MemOp->getSize().getValue();
3300 if (IsGPR && SrcSize == 32 && DstSize == 64) {
3302 MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3303 const Register ZReg = AArch64::WZR;
3304 MIB.
buildInstr(AArch64::ORRWrs, {SubregToRegSrc}, {ZReg, SrcReg})
3307 MIB.
buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
3310 .
addImm(AArch64::sub_32);
3312 if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
3314 LLVM_DEBUG(
dbgs() <<
"Failed to constrain G_ZEXT destination\n");
3318 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
3324 I.eraseFromParent();
3329 if (DstSize == 64) {
3330 if (Opcode != TargetOpcode::G_SEXT_INREG) {
3332 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
3338 SrcReg = MIB.
buildInstr(AArch64::SUBREG_TO_REG,
3339 {&AArch64::GPR64RegClass}, {})
3346 ExtI = MIB.
buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
3350 }
else if (DstSize <= 32) {
3351 ExtI = MIB.
buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
3360 I.eraseFromParent();
3364 case TargetOpcode::G_SITOFP:
3365 case TargetOpcode::G_UITOFP:
3366 case TargetOpcode::G_FPTOSI:
3367 case TargetOpcode::G_FPTOUI: {
3368 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg()),
3369 SrcTy =
MRI.getType(
I.getOperand(1).getReg());
3371 if (NewOpc == Opcode)
3374 I.setDesc(
TII.get(NewOpc));
3381 case TargetOpcode::G_FREEZE:
3384 case TargetOpcode::G_INTTOPTR:
3389 case TargetOpcode::G_BITCAST:
3397 case TargetOpcode::G_SELECT: {
3398 auto &Sel = cast<GSelect>(
I);
3399 const Register CondReg = Sel.getCondReg();
3400 const Register TReg = Sel.getTrueReg();
3401 const Register FReg = Sel.getFalseReg();
3403 if (tryOptSelect(Sel))
3408 Register DeadVReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3409 auto TstMI = MIB.
buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
3412 if (!emitSelect(Sel.getReg(0), TReg, FReg,
AArch64CC::NE, MIB))
3414 Sel.eraseFromParent();
3417 case TargetOpcode::G_ICMP: {
3430 emitIntegerCompare(
I.getOperand(2),
I.getOperand(3),
I.getOperand(1), MIB);
3431 emitCSINC(
I.getOperand(0).getReg(), AArch64::WZR,
3432 AArch64::WZR, InvCC, MIB);
3433 I.eraseFromParent();
3437 case TargetOpcode::G_FCMP: {
3440 if (!emitFPCompare(
I.getOperand(2).getReg(),
I.getOperand(3).getReg(), MIB,
3442 !emitCSetForFCmp(
I.getOperand(0).getReg(), Pred, MIB))
3444 I.eraseFromParent();
3447 case TargetOpcode::G_VASTART:
3448 return STI.isTargetDarwin() ? selectVaStartDarwin(
I, MF,
MRI)
3449 : selectVaStartAAPCS(
I, MF,
MRI);
3450 case TargetOpcode::G_INTRINSIC:
3451 return selectIntrinsic(
I,
MRI);
3452 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
3453 return selectIntrinsicWithSideEffects(
I,
MRI);
3454 case TargetOpcode::G_IMPLICIT_DEF: {
3455 I.setDesc(
TII.get(TargetOpcode::IMPLICIT_DEF));
3456 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
3457 const Register DstReg =
I.getOperand(0).getReg();
3460 RBI.constrainGenericRegister(DstReg, *DstRC,
MRI);
3463 case TargetOpcode::G_BLOCK_ADDR: {
3465 materializeLargeCMVal(
I,
I.getOperand(1).getBlockAddress(), 0);
3466 I.eraseFromParent();
3469 I.setDesc(
TII.get(AArch64::MOVaddrBA));
3470 auto MovMI =
BuildMI(
MBB,
I,
I.getDebugLoc(),
TII.get(AArch64::MOVaddrBA),
3471 I.getOperand(0).getReg())
3475 I.getOperand(1).getBlockAddress(), 0,
3477 I.eraseFromParent();
3481 case AArch64::G_DUP: {
3487 if (RBI.getRegBank(
I.getOperand(1).getReg(),
MRI,
TRI)->getID() !=
3488 AArch64::GPRRegBankID)
3490 LLT VecTy =
MRI.getType(
I.getOperand(0).getReg());
3492 I.setDesc(
TII.get(AArch64::DUPv8i8gpr));
3494 I.setDesc(
TII.get(AArch64::DUPv16i8gpr));
3496 I.setDesc(
TII.get(AArch64::DUPv4i16gpr));
3498 I.setDesc(
TII.get(AArch64::DUPv8i16gpr));
3503 case TargetOpcode::G_BUILD_VECTOR:
3504 return selectBuildVector(
I,
MRI);
3505 case TargetOpcode::G_MERGE_VALUES:
3507 case TargetOpcode::G_UNMERGE_VALUES:
3509 case TargetOpcode::G_SHUFFLE_VECTOR:
3510 return selectShuffleVector(
I,
MRI);
3511 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3512 return selectExtractElt(
I,
MRI);
3513 case TargetOpcode::G_CONCAT_VECTORS:
3514 return selectConcatVectors(
I,
MRI);
3515 case TargetOpcode::G_JUMP_TABLE:
3516 return selectJumpTable(
I,
MRI);
3517 case TargetOpcode::G_MEMCPY:
3518 case TargetOpcode::G_MEMCPY_INLINE:
3519 case TargetOpcode::G_MEMMOVE:
3520 case TargetOpcode::G_MEMSET:
3521 assert(STI.hasMOPS() &&
"Shouldn't get here without +mops feature");
3522 return selectMOPS(
I,
MRI);
3528bool AArch64InstructionSelector::selectAndRestoreState(
MachineInstr &
I) {
3535bool AArch64InstructionSelector::selectMOPS(
MachineInstr &GI,
3539 case TargetOpcode::G_MEMCPY:
3540 case TargetOpcode::G_MEMCPY_INLINE:
3541 Mopcode = AArch64::MOPSMemoryCopyPseudo;
3543 case TargetOpcode::G_MEMMOVE:
3544 Mopcode = AArch64::MOPSMemoryMovePseudo;
3546 case TargetOpcode::G_MEMSET:
3548 Mopcode = AArch64::MOPSMemorySetPseudo;
3557 const Register DstPtrCopy =
MRI.cloneVirtualRegister(DstPtr.getReg());
3558 const Register SrcValCopy =
MRI.cloneVirtualRegister(SrcOrVal.getReg());
3561 const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
3562 const auto &SrcValRegClass =
3563 IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;
3566 RBI.constrainGenericRegister(DstPtrCopy, AArch64::GPR64commonRegClass,
MRI);
3567 RBI.constrainGenericRegister(SrcValCopy, SrcValRegClass,
MRI);
3568 RBI.constrainGenericRegister(SizeCopy, AArch64::GPR64RegClass,
MRI);
3578 Register DefDstPtr =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
3579 Register DefSize =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3581 MIB.
buildInstr(Mopcode, {DefDstPtr, DefSize},
3582 {DstPtrCopy, SizeCopy, SrcValCopy});
3584 Register DefSrcPtr =
MRI.createVirtualRegister(&SrcValRegClass);
3585 MIB.
buildInstr(Mopcode, {DefDstPtr, DefSrcPtr, DefSize},
3586 {DstPtrCopy, SrcValCopy, SizeCopy});
3595 assert(
I.getOpcode() == TargetOpcode::G_BRJT &&
"Expected G_BRJT");
3596 Register JTAddr =
I.getOperand(0).getReg();
3597 unsigned JTI =
I.getOperand(1).getIndex();
3600 Register TargetReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3601 Register ScratchReg =
MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
3604 auto JumpTableInst = MIB.
buildInstr(AArch64::JumpTableDest32,
3605 {TargetReg, ScratchReg}, {JTAddr,
Index})
3606 .addJumpTableIndex(JTI);
3608 MIB.
buildInstr(TargetOpcode::JUMP_TABLE_DEBUG_INFO, {},
3609 {
static_cast<int64_t
>(JTI)});
3611 MIB.
buildInstr(AArch64::BR, {}, {TargetReg});
3612 I.eraseFromParent();
3616bool AArch64InstructionSelector::selectJumpTable(
MachineInstr &
I,
3618 assert(
I.getOpcode() == TargetOpcode::G_JUMP_TABLE &&
"Expected jump table");
3619 assert(
I.getOperand(1).isJTI() &&
"Jump table op should have a JTI!");
3621 Register DstReg =
I.getOperand(0).getReg();
3622 unsigned JTI =
I.getOperand(1).getIndex();
3625 MIB.
buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
3628 I.eraseFromParent();
3632bool AArch64InstructionSelector::selectTLSGlobalValue(
3634 if (!STI.isTargetMachO())
3639 const auto &GlobalOp =
I.getOperand(1);
3640 assert(GlobalOp.getOffset() == 0 &&
3641 "Shouldn't have an offset on TLS globals!");
3645 MIB.
buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
3648 auto Load = MIB.
buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
3649 {LoadGOT.getReg(0)})
3662 RBI.constrainGenericRegister(
I.getOperand(0).getReg(), AArch64::GPR64RegClass,
3664 I.eraseFromParent();
3668MachineInstr *AArch64InstructionSelector::emitScalarToVector(
3671 auto Undef = MIRBuilder.
buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});
3673 auto BuildFn = [&](
unsigned SubregIndex) {
3677 .addImm(SubregIndex);
3685 return BuildFn(AArch64::bsub);
3687 return BuildFn(AArch64::hsub);
3689 return BuildFn(AArch64::ssub);
3691 return BuildFn(AArch64::dsub);
3698AArch64InstructionSelector::emitNarrowVector(
Register DstReg,
Register SrcReg,
3701 LLT DstTy =
MRI.getType(DstReg);
3703 getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(SrcReg,
MRI,
TRI));
3704 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
3711 if (
SubReg != AArch64::ssub &&
SubReg != AArch64::dsub) {
3717 .addReg(SrcReg, 0,
SubReg);
3718 RBI.constrainGenericRegister(DstReg, *RC,
MRI);
3722bool AArch64InstructionSelector::selectMergeValues(
3724 assert(
I.getOpcode() == TargetOpcode::G_MERGE_VALUES &&
"unexpected opcode");
3725 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
3726 const LLT SrcTy =
MRI.getType(
I.getOperand(1).getReg());
3730 if (
I.getNumOperands() != 3)
3737 Register DstReg =
I.getOperand(0).getReg();
3738 Register Src1Reg =
I.getOperand(1).getReg();
3739 Register Src2Reg =
I.getOperand(2).getReg();
3740 auto Tmp = MIB.
buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
3741 MachineInstr *InsMI = emitLaneInsert(std::nullopt, Tmp.getReg(0), Src1Reg,
3746 Src2Reg, 1, RB, MIB);
3751 I.eraseFromParent();
3755 if (RB.
getID() != AArch64::GPRRegBankID)
3761 auto *DstRC = &AArch64::GPR64RegClass;
3762 Register SubToRegDef =
MRI.createVirtualRegister(DstRC);
3764 TII.get(TargetOpcode::SUBREG_TO_REG))
3767 .
addUse(
I.getOperand(1).getReg())
3768 .
addImm(AArch64::sub_32);
3769 Register SubToRegDef2 =
MRI.createVirtualRegister(DstRC);
3772 TII.get(TargetOpcode::SUBREG_TO_REG))
3775 .
addUse(
I.getOperand(2).getReg())
3776 .
addImm(AArch64::sub_32);
3778 *
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(AArch64::BFMXri))
3779 .
addDef(
I.getOperand(0).getReg())
3787 I.eraseFromParent();
3792 const unsigned EltSize) {
3797 CopyOpc = AArch64::DUPi8;
3798 ExtractSubReg = AArch64::bsub;
3801 CopyOpc = AArch64::DUPi16;
3802 ExtractSubReg = AArch64::hsub;
3805 CopyOpc = AArch64::DUPi32;
3806 ExtractSubReg = AArch64::ssub;
3809 CopyOpc = AArch64::DUPi64;
3810 ExtractSubReg = AArch64::dsub;
3814 LLVM_DEBUG(
dbgs() <<
"Elt size '" << EltSize <<
"' unsupported.\n");
3820MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
3821 std::optional<Register> DstReg,
const RegisterBank &DstRB,
LLT ScalarTy,
3824 unsigned CopyOpc = 0;
3825 unsigned ExtractSubReg = 0;
3828 dbgs() <<
"Couldn't determine lane copy opcode for instruction.\n");
3833 getRegClassForTypeOnBank(ScalarTy, DstRB,
true);
3835 LLVM_DEBUG(
dbgs() <<
"Could not determine destination register class.\n");
3840 const LLT &VecTy =
MRI.getType(VecReg);
3842 getRegClassForTypeOnBank(VecTy, VecRB,
true);
3844 LLVM_DEBUG(
dbgs() <<
"Could not determine source register class.\n");
3851 DstReg =
MRI.createVirtualRegister(DstRC);
3854 auto Copy = MIRBuilder.
buildInstr(TargetOpcode::COPY, {*DstReg}, {})
3855 .addReg(VecReg, 0, ExtractSubReg);
3856 RBI.constrainGenericRegister(*DstReg, *DstRC,
MRI);
3865 VecTy.
getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
3866 if (!ScalarToVector)
3872 MIRBuilder.
buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
3876 RBI.constrainGenericRegister(*DstReg, *DstRC,
MRI);
3880bool AArch64InstructionSelector::selectExtractElt(
3882 assert(
I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
3883 "unexpected opcode!");
3884 Register DstReg =
I.getOperand(0).getReg();
3885 const LLT NarrowTy =
MRI.getType(DstReg);
3886 const Register SrcReg =
I.getOperand(1).getReg();
3887 const LLT WideTy =
MRI.getType(SrcReg);
3890 "source register size too small!");
3891 assert(!NarrowTy.
isVector() &&
"cannot extract vector into vector!");
3895 assert(LaneIdxOp.
isReg() &&
"Lane index operand was not a register?");
3897 if (RBI.getRegBank(DstReg,
MRI,
TRI)->getID() != AArch64::FPRRegBankID) {
3906 unsigned LaneIdx = VRegAndVal->Value.getSExtValue();
3910 MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
3915 I.eraseFromParent();
3919bool AArch64InstructionSelector::selectSplitVectorUnmerge(
3921 unsigned NumElts =
I.getNumOperands() - 1;
3922 Register SrcReg =
I.getOperand(NumElts).getReg();
3923 const LLT NarrowTy =
MRI.getType(
I.getOperand(0).getReg());
3924 const LLT SrcTy =
MRI.getType(SrcReg);
3926 assert(NarrowTy.
isVector() &&
"Expected an unmerge into vectors");
3928 LLVM_DEBUG(
dbgs() <<
"Unexpected vector type for vec split unmerge");
3935 *RBI.getRegBank(
I.getOperand(0).getReg(),
MRI,
TRI);
3936 for (
unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
3937 Register Dst =
I.getOperand(OpIdx).getReg();
3939 emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
3943 I.eraseFromParent();
3947bool AArch64InstructionSelector::selectUnmergeValues(
MachineInstr &
I,
3949 assert(
I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
3950 "unexpected opcode");
3953 if (RBI.getRegBank(
I.getOperand(0).getReg(),
MRI,
TRI)->getID() !=
3954 AArch64::FPRRegBankID ||
3955 RBI.getRegBank(
I.getOperand(1).getReg(),
MRI,
TRI)->getID() !=
3956 AArch64::FPRRegBankID) {
3957 LLVM_DEBUG(
dbgs() <<
"Unmerging vector-to-gpr and scalar-to-scalar "
3958 "currently unsupported.\n");
3964 unsigned NumElts =
I.getNumOperands() - 1;
3965 Register SrcReg =
I.getOperand(NumElts).getReg();
3966 const LLT NarrowTy =
MRI.getType(
I.getOperand(0).getReg());
3967 const LLT WideTy =
MRI.getType(SrcReg);
3970 "can only unmerge from vector or s128 types!");
3972 "source register size too small!");
3975 return selectSplitVectorUnmerge(
I,
MRI);
3979 unsigned CopyOpc = 0;
3980 unsigned ExtractSubReg = 0;
3991 unsigned NumInsertRegs = NumElts - 1;
4003 *RBI.getRegBank(SrcReg,
MRI,
TRI));
4007 assert(Found &&
"expected to find last operand's subeg idx");
4008 for (
unsigned Idx = 0;
Idx < NumInsertRegs; ++
Idx) {
4009 Register ImpDefReg =
MRI.createVirtualRegister(&AArch64::FPR128RegClass);
4011 *
BuildMI(
MBB,
I,
I.getDebugLoc(),
TII.get(TargetOpcode::IMPLICIT_DEF),
4015 Register InsertReg =
MRI.createVirtualRegister(&AArch64::FPR128RegClass);
4018 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
4035 Register CopyTo =
I.getOperand(0).getReg();
4036 auto FirstCopy = MIB.
buildInstr(TargetOpcode::COPY, {CopyTo}, {})
4037 .addReg(InsertRegs[0], 0, ExtractSubReg);
4041 unsigned LaneIdx = 1;
4042 for (
Register InsReg : InsertRegs) {
4043 Register CopyTo =
I.getOperand(LaneIdx).getReg();
4056 MRI.getRegClassOrNull(
I.getOperand(1).getReg());
4062 RBI.constrainGenericRegister(CopyTo, *RC,
MRI);
4063 I.eraseFromParent();
4067bool AArch64InstructionSelector::selectConcatVectors(
4069 assert(
I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
4070 "Unexpected opcode");
4071 Register Dst =
I.getOperand(0).getReg();
4072 Register Op1 =
I.getOperand(1).getReg();
4073 Register Op2 =
I.getOperand(2).getReg();
4074 MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIB);
4077 I.eraseFromParent();
4082AArch64InstructionSelector::emitConstantPoolEntry(
const Constant *CPVal,
4091MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
4099 RC = &AArch64::FPR128RegClass;
4100 Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
4103 RC = &AArch64::FPR64RegClass;
4104 Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
4107 RC = &AArch64::FPR32RegClass;
4108 Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
4111 RC = &AArch64::FPR16RegClass;
4112 Opc = AArch64::LDRHui;
4115 LLVM_DEBUG(
dbgs() <<
"Could not load from constant pool of type "
4121 auto &MF = MIRBuilder.
getMF();
4122 unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
4123 if (IsTiny && (
Size == 16 ||
Size == 8 ||
Size == 4)) {
4125 LoadMI = &*MIRBuilder.
buildInstr(Opc, {RC}, {}).addConstantPoolIndex(CPIdx);
4128 MIRBuilder.
buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
4131 LoadMI = &*MIRBuilder.
buildInstr(Opc, {RC}, {Adrp})
4132 .addConstantPoolIndex(
4148static std::pair<unsigned, unsigned>
4150 unsigned Opc, SubregIdx;
4151 if (RB.
getID() == AArch64::GPRRegBankID) {
4153 Opc = AArch64::INSvi8gpr;
4154 SubregIdx = AArch64::bsub;
4155 }
else if (EltSize == 16) {
4156 Opc = AArch64::INSvi16gpr;
4157 SubregIdx = AArch64::ssub;
4158 }
else if (EltSize == 32) {
4159 Opc = AArch64::INSvi32gpr;
4160 SubregIdx = AArch64::ssub;
4161 }
else if (EltSize == 64) {
4162 Opc = AArch64::INSvi64gpr;
4163 SubregIdx = AArch64::dsub;
4169 Opc = AArch64::INSvi8lane;
4170 SubregIdx = AArch64::bsub;
4171 }
else if (EltSize == 16) {
4172 Opc = AArch64::INSvi16lane;
4173 SubregIdx = AArch64::hsub;
4174 }
else if (EltSize == 32) {
4175 Opc = AArch64::INSvi32lane;
4176 SubregIdx = AArch64::ssub;
4177 }
else if (EltSize == 64) {
4178 Opc = AArch64::INSvi64lane;
4179 SubregIdx = AArch64::dsub;
4184 return std::make_pair(Opc, SubregIdx);
4188 unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
4190 const ComplexRendererFns &RenderFns)
const {
4191 assert(Opcode &&
"Expected an opcode?");
4193 "Function should only be used to produce selected instructions!");
4194 auto MI = MIRBuilder.
buildInstr(Opcode, DstOps, SrcOps);
4196 for (
auto &Fn : *RenderFns)
4203 const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
4207 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4208 auto Ty =
MRI.getType(
LHS.getReg());
4211 assert((
Size == 32 ||
Size == 64) &&
"Expected a 32-bit or 64-bit type only");
4212 bool Is32Bit =
Size == 32;
4215 if (
auto Fns = selectArithImmed(RHS))
4216 return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {
LHS},
4220 if (
auto Fns = selectNegArithImmed(RHS))
4221 return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {
LHS},
4225 if (
auto Fns = selectArithExtendedRegister(RHS))
4226 return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {
LHS},
4230 if (
auto Fns = selectShiftedRegister(RHS))
4231 return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {
LHS},
4233 return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {
LHS,
RHS},
4241 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4242 {{AArch64::ADDXri, AArch64::ADDWri},
4243 {AArch64::ADDXrs, AArch64::ADDWrs},
4244 {AArch64::ADDXrr, AArch64::ADDWrr},
4245 {AArch64::SUBXri, AArch64::SUBWri},
4246 {AArch64::ADDXrx, AArch64::ADDWrx}}};
4247 return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder);
4254 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4255 {{AArch64::ADDSXri, AArch64::ADDSWri},
4256 {AArch64::ADDSXrs, AArch64::ADDSWrs},
4257 {AArch64::ADDSXrr, AArch64::ADDSWrr},
4258 {AArch64::SUBSXri, AArch64::SUBSWri},
4259 {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
4260 return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4267 const std::array<std::array<unsigned, 2>, 5> OpcTable{
4268 {{AArch64::SUBSXri, AArch64::SUBSWri},
4269 {AArch64::SUBSXrs, AArch64::SUBSWrs},
4270 {AArch64::SUBSXrr, AArch64::SUBSWrr},
4271 {AArch64::ADDSXri, AArch64::ADDSWri},
4272 {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
4273 return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4280 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4282 bool Is32Bit = (
MRI->getType(
LHS.getReg()).getSizeInBits() == 32);
4283 static const unsigned OpcTable[2] = {AArch64::ADCSXr, AArch64::ADCSWr};
4284 return emitInstr(OpcTable[Is32Bit], {Dst}, {
LHS,
RHS}, MIRBuilder);
4291 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4293 bool Is32Bit = (
MRI->getType(
LHS.getReg()).getSizeInBits() == 32);
4294 static const unsigned OpcTable[2] = {AArch64::SBCSXr, AArch64::SBCSWr};
4295 return emitInstr(OpcTable[Is32Bit], {Dst}, {
LHS,
RHS}, MIRBuilder);
4302 bool Is32Bit = (
MRI.getType(
LHS.getReg()).getSizeInBits() == 32);
4303 auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
4304 return emitADDS(
MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4310 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected register operands?");
4314 bool Is32Bit = (
RegSize == 32);
4315 const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
4316 {AArch64::ANDSXrs, AArch64::ANDSWrs},
4317 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
4321 int64_t
Imm = ValAndVReg->Value.getSExtValue();
4324 auto TstMI = MIRBuilder.
buildInstr(OpcTable[0][Is32Bit], {Ty}, {
LHS});
4331 if (
auto Fns = selectLogicalShiftedRegister(RHS))
4332 return emitInstr(OpcTable[1][Is32Bit], {Ty}, {
LHS}, MIRBuilder, Fns);
4333 return emitInstr(OpcTable[2][Is32Bit], {Ty}, {
LHS,
RHS}, MIRBuilder);
4336MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
4339 assert(
LHS.isReg() &&
RHS.isReg() &&
"Expected LHS and RHS to be registers!");
4346 assert((
Size == 32 ||
Size == 64) &&
"Expected a 32-bit or 64-bit LHS/RHS?");
4348 if (
auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
4350 auto Dst =
MRI.cloneVirtualRegister(
LHS.getReg());
4351 return emitSUBS(Dst, LHS, RHS, MIRBuilder);
4354MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
4358 LLT Ty =
MRI.getType(Dst);
4360 "Expected a 32-bit scalar register?");
4362 const Register ZReg = AArch64::WZR;
4367 return emitCSINC(Dst, ZReg, ZReg, InvCC1,
4373 emitCSINC(Def1Reg, ZReg, ZReg, InvCC1, MIRBuilder);
4374 emitCSINC(Def2Reg, ZReg, ZReg, InvCC2, MIRBuilder);
4375 auto OrMI = MIRBuilder.
buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
4380MachineInstr *AArch64InstructionSelector::emitFPCompare(
4382 std::optional<CmpInst::Predicate> Pred)
const {
4384 LLT Ty =
MRI.getType(LHS);
4388 assert(OpSize == 16 || OpSize == 32 || OpSize == 64);
4399 if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
4403 ShouldUseImm =
true;
4407 unsigned CmpOpcTbl[2][3] = {
4408 {AArch64::FCMPHrr, AArch64::FCMPSrr, AArch64::FCMPDrr},
4409 {AArch64::FCMPHri, AArch64::FCMPSri, AArch64::FCMPDri}};
4411 CmpOpcTbl[ShouldUseImm][OpSize == 16 ? 0 : (OpSize == 32 ? 1 : 2)];
4423MachineInstr *AArch64InstructionSelector::emitVectorConcat(
4432 const LLT Op1Ty =
MRI.getType(Op1);
4433 const LLT Op2Ty =
MRI.getType(Op2);
4435 if (Op1Ty != Op2Ty) {
4436 LLVM_DEBUG(
dbgs() <<
"Could not do vector concat of differing vector tys");
4439 assert(Op1Ty.
isVector() &&
"Expected a vector for vector concat");
4442 LLVM_DEBUG(
dbgs() <<
"Vector concat not supported for full size vectors");
4458 emitScalarToVector(ScalarTy.
getSizeInBits(), DstRC, Op1, MIRBuilder);
4460 emitScalarToVector(ScalarTy.
getSizeInBits(), DstRC, Op2, MIRBuilder);
4461 if (!WidenedOp1 || !WidenedOp2) {
4462 LLVM_DEBUG(
dbgs() <<
"Could not emit a vector from scalar value");
4467 unsigned InsertOpc, InsSubRegIdx;
4468 std::tie(InsertOpc, InsSubRegIdx) =
4472 Dst =
MRI.createVirtualRegister(DstRC);
4493 Size =
TRI.getRegSizeInBits(*RC);
4495 Size =
MRI.getType(Dst).getSizeInBits();
4497 assert(
Size <= 64 &&
"Expected 64 bits or less only!");
4498 static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
4499 unsigned Opc = OpcTable[
Size == 64];
4500 auto CSINC = MIRBuilder.
buildInstr(Opc, {Dst}, {Src1, Src2}).addImm(Pred);
4508 unsigned Opcode =
I.getOpcode();
4512 bool NeedsNegatedCarry =
4513 (Opcode == TargetOpcode::G_USUBE || Opcode == TargetOpcode::G_SSUBE);
4523 if (SrcMI ==
I.getPrevNode()) {
4524 if (
auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(SrcMI)) {
4525 bool ProducesNegatedCarry = CarrySrcMI->isSub();
4526 if (NeedsNegatedCarry == ProducesNegatedCarry &&
4527 CarrySrcMI->isUnsigned() &&
4528 CarrySrcMI->getCarryOutReg() == CarryReg &&
4529 selectAndRestoreState(*SrcMI))
4534 Register DeadReg =
MRI->createVirtualRegister(&AArch64::GPR32RegClass);
4536 if (NeedsNegatedCarry) {
4539 return emitInstr(AArch64::SUBSWrr, {DeadReg}, {ZReg, CarryReg}, MIB);
4543 auto Fns = select12BitValueWithLeftShift(1);
4544 return emitInstr(AArch64::SUBSWri, {DeadReg}, {CarryReg}, MIB, Fns);
4547bool AArch64InstructionSelector::selectOverflowOp(
MachineInstr &
I,
4549 auto &CarryMI = cast<GAddSubCarryOut>(
I);
4551 if (
auto *CarryInMI = dyn_cast<GAddSubCarryInOut>(&
I)) {
4553 emitCarryIn(
I, CarryInMI->getCarryInReg());
4557 auto OpAndCC = emitOverflowOp(
I.getOpcode(), CarryMI.getDstReg(),
4558 CarryMI.getLHS(), CarryMI.getRHS(), MIB);
4560 Register CarryOutReg = CarryMI.getCarryOutReg();
4563 if (!
MRI.use_nodbg_empty(CarryOutReg)) {
4569 emitCSINC(CarryOutReg, ZReg, ZReg,
4570 getInvertedCondCode(OpAndCC.second), MIB);
4573 I.eraseFromParent();
4577std::pair<MachineInstr *, AArch64CC::CondCode>
4578AArch64InstructionSelector::emitOverflowOp(
unsigned Opcode,
Register Dst,
4585 case TargetOpcode::G_SADDO:
4586 return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::VS);
4587 case TargetOpcode::G_UADDO:
4588 return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::HS);
4589 case TargetOpcode::G_SSUBO:
4590 return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::VS);
4591 case TargetOpcode::G_USUBO:
4592 return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::LO);
4593 case TargetOpcode::G_SADDE:
4594 return std::make_pair(emitADCS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::VS);
4595 case TargetOpcode::G_UADDE:
4596 return std::make_pair(emitADCS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::HS);
4597 case TargetOpcode::G_SSUBE:
4598 return std::make_pair(emitSBCS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::VS);
4599 case TargetOpcode::G_USUBE:
4600 return std::make_pair(emitSBCS(Dst, LHS, RHS, MIRBuilder),
AArch64CC::LO);
4620 unsigned Depth = 0) {
4621 if (!
MRI.hasOneNonDBGUse(Val))
4625 if (isa<GAnyCmp>(ValDef)) {
4627 MustBeFirst =
false;
4633 if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
4634 bool IsOR = Opcode == TargetOpcode::G_OR;
4646 if (MustBeFirstL && MustBeFirstR)
4652 if (!CanNegateL && !CanNegateR)
4656 CanNegate = WillNegate && CanNegateL && CanNegateR;
4659 MustBeFirst = !CanNegate;
4661 assert(Opcode == TargetOpcode::G_AND &&
"Must be G_AND");
4664 MustBeFirst = MustBeFirstL || MustBeFirstR;
4671MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
4676 LLT OpTy =
MRI.getType(LHS);
4678 std::optional<ValueAndVReg>
C;
4682 if (!
C ||
C->Value.sgt(31) ||
C->Value.slt(-31))
4683 CCmpOpc = OpTy.
getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
4684 else if (
C->Value.ule(31))
4685 CCmpOpc = OpTy.
getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
4687 CCmpOpc = OpTy.
getSizeInBits() == 32 ? AArch64::CCMNWi : AArch64::CCMNXi;
4693 assert(STI.hasFullFP16() &&
"Expected Full FP16 for fp16 comparisons");
4694 CCmpOpc = AArch64::FCCMPHrr;
4697 CCmpOpc = AArch64::FCCMPSrr;
4700 CCmpOpc = AArch64::FCCMPDrr;
4710 if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
4711 CCmp.
addImm(
C->Value.getZExtValue());
4712 else if (CCmpOpc == AArch64::CCMNWi || CCmpOpc == AArch64::CCMNXi)
4713 CCmp.
addImm(
C->Value.abs().getZExtValue());
4721MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
4728 if (
auto *Cmp = dyn_cast<GAnyCmp>(ValDef)) {
4734 if (isa<GICmp>(Cmp)) {
4745 ExtraCmp = emitFPCompare(LHS, RHS, MIB,
CC);
4756 auto Dst =
MRI.cloneVirtualRegister(LHS);
4757 if (isa<GICmp>(Cmp))
4758 return emitSUBS(Dst,
Cmp->getOperand(2),
Cmp->getOperand(3), MIB);
4759 return emitFPCompare(
Cmp->getOperand(2).getReg(),
4760 Cmp->getOperand(3).getReg(), MIB);
4765 assert(
MRI.hasOneNonDBGUse(Val) &&
"Valid conjunction/disjunction tree");
4767 bool IsOR = Opcode == TargetOpcode::G_OR;
4773 assert(ValidL &&
"Valid conjunction/disjunction tree");
4780 assert(ValidR &&
"Valid conjunction/disjunction tree");
4785 assert(!MustBeFirstR &&
"Valid conjunction/disjunction tree");
4794 bool NegateAfterAll;
4795 if (Opcode == TargetOpcode::G_OR) {
4798 assert(CanNegateR &&
"at least one side must be negatable");
4799 assert(!MustBeFirstR &&
"invalid conjunction/disjunction tree");
4803 NegateAfterR =
true;
4806 NegateR = CanNegateR;
4807 NegateAfterR = !CanNegateR;
4810 NegateAfterAll = !Negate;
4812 assert(Opcode == TargetOpcode::G_AND &&
4813 "Valid conjunction/disjunction tree");
4814 assert(!Negate &&
"Valid conjunction/disjunction tree");
4818 NegateAfterR =
false;
4819 NegateAfterAll =
false;
4835MachineInstr *AArch64InstructionSelector::emitConjunction(
4837 bool DummyCanNegate;
4838 bool DummyMustBeFirst;
4845bool AArch64InstructionSelector::tryOptSelectConjunction(
GSelect &SelI,
4857bool AArch64InstructionSelector::tryOptSelect(
GSelect &
I) {
4881 if (!
MRI.hasOneNonDBGUse(CondDefReg)) {
4883 for (
const MachineInstr &UI :
MRI.use_nodbg_instructions(CondDefReg)) {
4886 if (UI.getOpcode() != TargetOpcode::G_SELECT)
4892 unsigned CondOpc = CondDef->
getOpcode();
4893 if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
4894 if (tryOptSelectConjunction(
I, *CondDef))
4900 if (CondOpc == TargetOpcode::G_ICMP) {
4928 emitSelect(
I.getOperand(0).getReg(),
I.getOperand(2).getReg(),
4929 I.getOperand(3).getReg(), CondCode, MIB);
4930 I.eraseFromParent();
4934MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
4938 "Unexpected MachineOperand");
4975 return emitCMN(LHS, RHSDef->
getOperand(2), MIRBuilder);
4986 LHSDef->
getOpcode() == TargetOpcode::G_AND) {
4989 if (!ValAndVReg || ValAndVReg->Value != 0)
4999bool AArch64InstructionSelector::selectShuffleVector(
5001 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
5002 Register Src1Reg =
I.getOperand(1).getReg();
5003 const LLT Src1Ty =
MRI.getType(Src1Reg);
5004 Register Src2Reg =
I.getOperand(2).getReg();
5005 const LLT Src2Ty =
MRI.getType(Src2Reg);
5016 LLVM_DEBUG(
dbgs() <<
"Could not select a \"scalar\" G_SHUFFLE_VECTOR\n");
5023 for (
int Val : Mask) {
5026 Val = Val < 0 ? 0 : Val;
5027 for (
unsigned Byte = 0;
Byte < BytesPerElt; ++
Byte) {
5045 emitVectorConcat(std::nullopt, Src1Reg, Src2Reg, MIB);
5052 IndexLoad = emitScalarToVector(64, &AArch64::FPR128RegClass,
5056 AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
5062 .addReg(TBL1.getReg(0), 0, AArch64::dsub);
5063 RBI.constrainGenericRegister(
Copy.getReg(0), AArch64::FPR64RegClass,
MRI);
5064 I.eraseFromParent();
5072 auto TBL2 = MIB.
buildInstr(AArch64::TBLv16i8Two, {
I.getOperand(0)},
5075 I.eraseFromParent();
5079MachineInstr *AArch64InstructionSelector::emitLaneInsert(
5089 DstReg =
MRI.createVirtualRegister(DstRC);
5091 unsigned EltSize =
MRI.getType(EltReg).getSizeInBits();
5094 if (RB.
getID() == AArch64::FPRRegBankID) {
5095 auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
5096 InsElt = MIRBuilder.
buildInstr(Opc, {*DstReg}, {SrcReg})
5098 .
addUse(InsSub->getOperand(0).getReg())
5101 InsElt = MIRBuilder.
buildInstr(Opc, {*DstReg}, {SrcReg})
5110bool AArch64InstructionSelector::selectUSMovFromExtend(
5112 if (
MI.getOpcode() != TargetOpcode::G_SEXT &&
5113 MI.getOpcode() != TargetOpcode::G_ZEXT &&
5114 MI.getOpcode() != TargetOpcode::G_ANYEXT)
5116 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SEXT;
5117 const Register DefReg =
MI.getOperand(0).getReg();
5118 const LLT DstTy =
MRI.getType(DefReg);
5121 if (DstSize != 32 && DstSize != 64)
5125 MI.getOperand(1).getReg(),
MRI);
5131 const LLT &VecTy =
MRI.getType(Src0);
5134 const MachineInstr *ScalarToVector = emitScalarToVector(
5135 VecTy.
getSizeInBits(), &AArch64::FPR128RegClass, Src0, MIB);
5136 assert(ScalarToVector &&
"Didn't expect emitScalarToVector to fail!");
5142 Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
5144 Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
5146 Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
5148 Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
5150 Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
5159 if (DstSize == 64 && !IsSigned) {
5160 Register NewReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
5161 MIB.
buildInstr(Opcode, {NewReg}, {Src0}).addImm(Lane);
5162 ExtI = MIB.
buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
5165 .
addImm(AArch64::sub_32);
5166 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
MRI);
5168 ExtI = MIB.
buildInstr(Opcode, {DefReg}, {Src0}).addImm(Lane);
5171 MI.eraseFromParent();
5175MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm8(
5178 if (DstSize == 128) {
5179 if (
Bits.getHiBits(64) !=
Bits.getLoBits(64))
5181 Op = AArch64::MOVIv16b_ns;
5183 Op = AArch64::MOVIv8b_ns;
5190 auto Mov = Builder.
buildInstr(
Op, {Dst}, {}).addImm(Val);
5197MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm16(
5202 if (DstSize == 128) {
5203 if (
Bits.getHiBits(64) !=
Bits.getLoBits(64))
5205 Op = Inv ? AArch64::MVNIv8i16 : AArch64::MOVIv8i16;
5207 Op = Inv ? AArch64::MVNIv4i16 : AArch64::MOVIv4i16;
5227MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm32(
5232 if (DstSize == 128) {
5233 if (
Bits.getHiBits(64) !=
Bits.getLoBits(64))
5235 Op = Inv ? AArch64::MVNIv4i32 : AArch64::MOVIv4i32;
5237 Op = Inv ? AArch64::MVNIv2i32 : AArch64::MOVIv2i32;
5263MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm64(
5267 if (DstSize == 128) {
5268 if (
Bits.getHiBits(64) !=
Bits.getLoBits(64))
5270 Op = AArch64::MOVIv2d_ns;
5272 Op = AArch64::MOVID;
5278 auto Mov = Builder.
buildInstr(
Op, {Dst}, {}).addImm(Val);
5285MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImm321s(
5290 if (DstSize == 128) {
5291 if (
Bits.getHiBits(64) !=
Bits.getLoBits(64))
5293 Op = Inv ? AArch64::MVNIv4s_msl : AArch64::MOVIv4s_msl;
5295 Op = Inv ? AArch64::MVNIv2s_msl : AArch64::MOVIv2s_msl;
5315MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImmFP(
5319 bool IsWide =
false;
5320 if (DstSize == 128) {
5321 if (
Bits.getHiBits(64) !=
Bits.getLoBits(64))
5323 Op = AArch64::FMOVv4f32_ns;
5326 Op = AArch64::FMOVv2f32_ns;
5335 Op = AArch64::FMOVv2f64_ns;
5339 auto Mov = Builder.
buildInstr(
Op, {Dst}, {}).addImm(Val);
5344bool AArch64InstructionSelector::selectIndexedExtLoad(
5346 auto &ExtLd = cast<GIndexedAnyExtLoad>(
MI);
5348 Register WriteBack = ExtLd.getWritebackReg();
5351 LLT Ty =
MRI.getType(Dst);
5353 unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
5354 bool IsPre = ExtLd.isPre();
5355 bool IsSExt = isa<GIndexedSExtLoad>(ExtLd);
5356 bool InsertIntoXReg =
false;
5364 if (MemSizeBits == 8) {
5367 Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
5369 Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
5370 NewLdDstTy = IsDst64 ? s64 : s32;
5372 Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
5373 InsertIntoXReg = IsDst64;
5376 }
else if (MemSizeBits == 16) {
5379 Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
5381 Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
5382 NewLdDstTy = IsDst64 ? s64 : s32;
5384 Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
5385 InsertIntoXReg = IsDst64;
5388 }
else if (MemSizeBits == 32) {
5390 Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
5393 Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
5394 InsertIntoXReg = IsDst64;
5401 if (RBI.getRegBank(Dst,
MRI,
TRI)->getID() == AArch64::FPRRegBankID)
5409 .addImm(Cst->getSExtValue());
5414 if (InsertIntoXReg) {
5416 auto SubToReg = MIB.
buildInstr(TargetOpcode::SUBREG_TO_REG, {Dst}, {})
5419 .
addImm(AArch64::sub_32);
5420 RBI.constrainGenericRegister(SubToReg.getReg(0), AArch64::GPR64RegClass,
5426 MI.eraseFromParent();
5431bool AArch64InstructionSelector::selectIndexedLoad(
MachineInstr &
MI,
5433 auto &Ld = cast<GIndexedLoad>(
MI);
5435 Register WriteBack = Ld.getWritebackReg();
5438 assert(
MRI.getType(Dst).getSizeInBits() <= 128 &&
5439 "Unexpected type for indexed load");
5440 unsigned MemSize = Ld.getMMO().getMemoryType().getSizeInBytes();
5442 if (MemSize <
MRI.getType(Dst).getSizeInBytes())
5443 return selectIndexedExtLoad(
MI,
MRI);
5447 static constexpr unsigned GPROpcodes[] = {
5448 AArch64::LDRBBpre, AArch64::LDRHHpre, AArch64::LDRWpre,
5450 static constexpr unsigned FPROpcodes[] = {
5451 AArch64::LDRBpre, AArch64::LDRHpre, AArch64::LDRSpre, AArch64::LDRDpre,
5453 if (RBI.getRegBank(Dst,
MRI,
TRI)->getID() == AArch64::FPRRegBankID)
5454 Opc = FPROpcodes[
Log2_32(MemSize)];
5456 Opc = GPROpcodes[
Log2_32(MemSize)];
5458 static constexpr unsigned GPROpcodes[] = {
5459 AArch64::LDRBBpost, AArch64::LDRHHpost, AArch64::LDRWpost,
5461 static constexpr unsigned FPROpcodes[] = {
5462 AArch64::LDRBpost, AArch64::LDRHpost, AArch64::LDRSpost,
5463 AArch64::LDRDpost, AArch64::LDRQpost};
5464 if (RBI.getRegBank(Dst,
MRI,
TRI)->getID() == AArch64::FPRRegBankID)
5465 Opc = FPROpcodes[
Log2_32(MemSize)];
5467 Opc = GPROpcodes[
Log2_32(MemSize)];
5473 MIB.
buildInstr(Opc, {WriteBack, Dst}, {
Base}).addImm(Cst->getSExtValue());
5476 MI.eraseFromParent();
5480bool AArch64InstructionSelector::selectIndexedStore(
GIndexedStore &
I,
5486 LLT ValTy =
MRI.getType(Val);
5491 static constexpr unsigned GPROpcodes[] = {
5492 AArch64::STRBBpre, AArch64::STRHHpre, AArch64::STRWpre,
5494 static constexpr unsigned FPROpcodes[] = {
5495 AArch64::STRBpre, AArch64::STRHpre, AArch64::STRSpre, AArch64::STRDpre,
5498 if (RBI.getRegBank(Val,
MRI,
TRI)->getID() == AArch64::FPRRegBankID)
5503 static constexpr unsigned GPROpcodes[] = {
5504 AArch64::STRBBpost, AArch64::STRHHpost, AArch64::STRWpost,
5506 static constexpr unsigned FPROpcodes[] = {
5507 AArch64::STRBpost, AArch64::STRHpost, AArch64::STRSpost,
5508 AArch64::STRDpost, AArch64::STRQpost};
5510 if (RBI.getRegBank(Val,
MRI,
TRI)->getID() == AArch64::FPRRegBankID)
5520 MIB.
buildInstr(Opc, {Dst}, {Val,
Base}).addImm(Cst->getSExtValue());
5521 Str.cloneMemRefs(
I);
5523 I.eraseFromParent();
5531 LLT DstTy =
MRI.getType(Dst);
5534 if (DstSize == 128) {
5536 MIRBuilder.
buildInstr(AArch64::MOVIv2d_ns, {Dst}, {}).addImm(0);
5541 if (DstSize == 64) {
5544 .
buildInstr(AArch64::MOVIv2d_ns, {&AArch64::FPR128RegClass}, {})
5547 .addReg(Mov.getReg(0), 0, AArch64::dsub);
5548 RBI.constrainGenericRegister(Dst, AArch64::FPR64RegClass,
MRI);
5580 if (
auto *NewOp = TryMOVIWithBits(DefBits))
5584 auto TryWithFNeg = [&](
APInt DefBits,
int NumBits,
5588 APInt NegBits(DstSize, 0);
5589 unsigned NumElts = DstSize / NumBits;
5590 for (
unsigned i = 0; i < NumElts; i++)
5591 NegBits |= Neg << (NumBits * i);
5592 NegBits = DefBits ^ NegBits;
5596 if (
auto *NewOp = TryMOVIWithBits(NegBits)) {
5597 Register NewDst =
MRI.createVirtualRegister(&AArch64::FPR128RegClass);
5599 return MIRBuilder.
buildInstr(NegOpc, {Dst}, {NewDst});
5604 if ((R = TryWithFNeg(DefBits, 32, AArch64::FNEGv4f32)) ||
5605 (R = TryWithFNeg(DefBits, 64, AArch64::FNEGv2f64)) ||
5606 (STI.hasFullFP16() &&
5607 (R = TryWithFNeg(DefBits, 16, AArch64::FNEGv8f16))))
5613 LLVM_DEBUG(
dbgs() <<
"Could not generate cp load for constant vector!");
5617 auto Copy = MIRBuilder.
buildCopy(Dst, CPLoad->getOperand(0));
5618 RBI.constrainGenericRegister(
5619 Dst, *
MRI.getRegClass(CPLoad->getOperand(0).getReg()),
MRI);
5623bool AArch64InstructionSelector::tryOptConstantBuildVec(
5625 assert(
I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5627 assert(DstSize <= 128 &&
"Unexpected build_vec type!");
5633 for (
unsigned Idx = 1;
Idx <
I.getNumOperands(); ++
Idx) {
5639 const_cast<ConstantInt *
>(OpMI->getOperand(1).getCImm()));
5640 else if ((OpMI =
getOpcodeDef(TargetOpcode::G_FCONSTANT,
5641 I.getOperand(
Idx).getReg(),
MRI)))
5643 const_cast<ConstantFP *
>(OpMI->getOperand(1).getFPImm()));
5648 if (!emitConstantVector(
I.getOperand(0).getReg(), CV, MIB,
MRI))
5650 I.eraseFromParent();
5654bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
5660 Register Dst =
I.getOperand(0).getReg();
5661 Register EltReg =
I.getOperand(1).getReg();
5662 LLT EltTy =
MRI.getType(EltReg);
5670 return !getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Op.getReg(), MRI);
5678 getRegClassForTypeOnBank(
MRI.getType(Dst), DstRB);
5683 auto SubregToReg = MIB.
buildInstr(AArch64::SUBREG_TO_REG, {Dst}, {})
5687 I.eraseFromParent();
5689 return RBI.constrainGenericRegister(Dst, *DstRC,
MRI);
5692bool AArch64InstructionSelector::selectBuildVector(
MachineInstr &
I,
5694 assert(
I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
5697 const LLT DstTy =
MRI.getType(
I.getOperand(0).getReg());
5698 const LLT EltTy =
MRI.getType(
I.getOperand(1).getReg());
5701 if (tryOptConstantBuildVec(
I, DstTy,
MRI))
5703 if (tryOptBuildVecToSubregToReg(
I,
MRI))
5706 if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
5713 I.getOperand(1).getReg(), MIB);
5723 for (
unsigned i = 2, e = DstSize / EltSize + 1; i <
e; ++i) {
5726 Register OpReg =
I.getOperand(i).getReg();
5728 if (!getOpcodeDef<GImplicitDef>(OpReg,
MRI)) {
5729 PrevMI = &*emitLaneInsert(std::nullopt, DstVec, OpReg, i - 1, RB, MIB);
5736 if (DstSize < 128) {
5739 getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(DstVec,
MRI,
TRI));
5742 if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
5750 if (
SubReg != AArch64::ssub &&
SubReg != AArch64::dsub) {
5751 LLVM_DEBUG(
dbgs() <<
"Unsupported destination size! (" << DstSize
5757 Register DstReg =
I.getOperand(0).getReg();
5759 MIB.
buildInstr(TargetOpcode::COPY, {DstReg}, {}).addReg(DstVec, 0,
SubReg);
5762 RBI.constrainGenericRegister(DstReg, *RC,
MRI);
5780 if (PrevMI == ScalarToVec && DstReg.
isVirtual()) {
5782 getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(DstVec,
MRI,
TRI));
5783 RBI.constrainGenericRegister(DstReg, *RC,
MRI);
5787 I.eraseFromParent();
5791bool AArch64InstructionSelector::selectVectorLoadIntrinsic(
unsigned Opc,
5794 assert(
I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
5795 assert(Opc &&
"Expected an opcode?");
5796 assert(NumVecs > 1 && NumVecs < 5 &&
"Only support 2, 3, or 4 vectors");
5798 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
5801 "Destination must be 64 bits or 128 bits?");
5802 unsigned SubReg =
Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
5803 auto Ptr =
I.getOperand(
I.getNumOperands() - 1).getReg();
5804 assert(
MRI.getType(
Ptr).isPointer() &&
"Expected a pointer type?");
5806 Load.cloneMemRefs(
I);
5808 Register SelectedLoadDst =
Load->getOperand(0).getReg();
5809 for (
unsigned Idx = 0;
Idx < NumVecs; ++
Idx) {
5810 auto Vec = MIB.
buildInstr(TargetOpcode::COPY, {
I.getOperand(
Idx)}, {})
5811 .addReg(SelectedLoadDst, 0,
SubReg +
Idx);
5820bool AArch64InstructionSelector::selectVectorLoadLaneIntrinsic(
5822 assert(
I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
5823 assert(Opc &&
"Expected an opcode?");
5824 assert(NumVecs > 1 && NumVecs < 5 &&
"Only support 2, 3, or 4 vectors");
5826 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
5829 auto FirstSrcRegIt =
I.operands_begin() + NumVecs + 1;
5831 std::transform(FirstSrcRegIt, FirstSrcRegIt + NumVecs, Regs.
begin(),
5832 [](
auto MO) { return MO.getReg(); });
5836 return emitScalarToVector(64, &AArch64::FPR128RegClass, Reg, MIB)
5851 .
addImm(LaneNo->getZExtValue())
5853 Load.cloneMemRefs(
I);
5855 Register SelectedLoadDst =
Load->getOperand(0).getReg();
5856 unsigned SubReg = AArch64::qsub0;
5857 for (
unsigned Idx = 0;
Idx < NumVecs; ++
Idx) {
5858 auto Vec = MIB.
buildInstr(TargetOpcode::COPY,
5859 {Narrow ?
DstOp(&AArch64::FPR128RegClass)
5862 .addReg(SelectedLoadDst, 0,
SubReg +
Idx);
5867 !emitNarrowVector(
I.getOperand(
Idx).getReg(), WideReg, MIB,
MRI))
5873void AArch64InstructionSelector::selectVectorStoreIntrinsic(
MachineInstr &
I,
5877 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
5881 std::transform(
I.operands_begin() + 1,
I.operands_begin() + 1 + NumVecs,
5882 Regs.
begin(), [](
auto MO) { return MO.getReg(); });
5891bool AArch64InstructionSelector::selectVectorStoreLaneIntrinsic(
5894 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
5898 std::transform(
I.operands_begin() + 1,
I.operands_begin() + 1 + NumVecs,
5899 Regs.
begin(), [](
auto MO) { return MO.getReg(); });
5903 return emitScalarToVector(64, &AArch64::FPR128RegClass, Reg, MIB)
5916 .
addImm(LaneNo->getZExtValue())
5923bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
5926 unsigned IntrinID = cast<GIntrinsic>(
I).getIntrinsicID();
5937 case Intrinsic::aarch64_ldxp:
5938 case Intrinsic::aarch64_ldaxp: {
5940 IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
5941 {
I.getOperand(0).
getReg(),
I.getOperand(1).getReg()},
5947 case Intrinsic::aarch64_neon_ld1x2: {
5948 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
5951 Opc = AArch64::LD1Twov8b;
5953 Opc = AArch64::LD1Twov16b;
5955 Opc = AArch64::LD1Twov4h;
5957 Opc = AArch64::LD1Twov8h;
5959 Opc = AArch64::LD1Twov2s;
5961 Opc = AArch64::LD1Twov4s;
5963 Opc = AArch64::LD1Twov2d;
5964 else if (Ty ==
S64 || Ty == P0)
5965 Opc = AArch64::LD1Twov1d;
5968 selectVectorLoadIntrinsic(Opc, 2,
I);
5971 case Intrinsic::aarch64_neon_ld1x3: {
5972 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
5975 Opc = AArch64::LD1Threev8b;
5977 Opc = AArch64::LD1Threev16b;
5979 Opc = AArch64::LD1Threev4h;
5981 Opc = AArch64::LD1Threev8h;
5983 Opc = AArch64::LD1Threev2s;
5985 Opc = AArch64::LD1Threev4s;
5987 Opc = AArch64::LD1Threev2d;
5988 else if (Ty ==
S64 || Ty == P0)
5989 Opc = AArch64::LD1Threev1d;
5992 selectVectorLoadIntrinsic(Opc, 3,
I);
5995 case Intrinsic::aarch64_neon_ld1x4: {
5996 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
5999 Opc = AArch64::LD1Fourv8b;
6001 Opc = AArch64::LD1Fourv16b;
6003 Opc = AArch64::LD1Fourv4h;
6005 Opc = AArch64::LD1Fourv8h;
6007 Opc = AArch64::LD1Fourv2s;
6009 Opc = AArch64::LD1Fourv4s;
6011 Opc = AArch64::LD1Fourv2d;
6012 else if (Ty ==
S64 || Ty == P0)
6013 Opc = AArch64::LD1Fourv1d;
6016 selectVectorLoadIntrinsic(Opc, 4,
I);
6019 case Intrinsic::aarch64_neon_ld2: {
6020 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6023 Opc = AArch64::LD2Twov8b;
6025 Opc = AArch64::LD2Twov16b;
6027 Opc = AArch64::LD2Twov4h;
6029 Opc = AArch64::LD2Twov8h;
6031 Opc = AArch64::LD2Twov2s;
6033 Opc = AArch64::LD2Twov4s;
6035 Opc = AArch64::LD2Twov2d;
6036 else if (Ty ==
S64 || Ty == P0)
6037 Opc = AArch64::LD1Twov1d;
6040 selectVectorLoadIntrinsic(Opc, 2,
I);
6043 case Intrinsic::aarch64_neon_ld2lane: {
6044 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6047 Opc = AArch64::LD2i8;
6049 Opc = AArch64::LD2i16;
6051 Opc = AArch64::LD2i32;
6054 Opc = AArch64::LD2i64;
6057 if (!selectVectorLoadLaneIntrinsic(Opc, 2,
I))
6061 case Intrinsic::aarch64_neon_ld2r: {
6062 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6065 Opc = AArch64::LD2Rv8b;
6067 Opc = AArch64::LD2Rv16b;
6069 Opc = AArch64::LD2Rv4h;
6071 Opc = AArch64::LD2Rv8h;
6073 Opc = AArch64::LD2Rv2s;
6075 Opc = AArch64::LD2Rv4s;
6077 Opc = AArch64::LD2Rv2d;
6078 else if (Ty ==
S64 || Ty == P0)
6079 Opc = AArch64::LD2Rv1d;
6082 selectVectorLoadIntrinsic(Opc, 2,
I);
6085 case Intrinsic::aarch64_neon_ld3: {
6086 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6089 Opc = AArch64::LD3Threev8b;
6091 Opc = AArch64::LD3Threev16b;
6093 Opc = AArch64::LD3Threev4h;
6095 Opc = AArch64::LD3Threev8h;
6097 Opc = AArch64::LD3Threev2s;
6099 Opc = AArch64::LD3Threev4s;
6101 Opc = AArch64::LD3Threev2d;
6102 else if (Ty ==
S64 || Ty == P0)
6103 Opc = AArch64::LD1Threev1d;
6106 selectVectorLoadIntrinsic(Opc, 3,
I);
6109 case Intrinsic::aarch64_neon_ld3lane: {
6110 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6113 Opc = AArch64::LD3i8;
6115 Opc = AArch64::LD3i16;
6117 Opc = AArch64::LD3i32;
6120 Opc = AArch64::LD3i64;
6123 if (!selectVectorLoadLaneIntrinsic(Opc, 3,
I))
6127 case Intrinsic::aarch64_neon_ld3r: {
6128 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6131 Opc = AArch64::LD3Rv8b;
6133 Opc = AArch64::LD3Rv16b;
6135 Opc = AArch64::LD3Rv4h;
6137 Opc = AArch64::LD3Rv8h;
6139 Opc = AArch64::LD3Rv2s;
6141 Opc = AArch64::LD3Rv4s;
6143 Opc = AArch64::LD3Rv2d;
6144 else if (Ty ==
S64 || Ty == P0)
6145 Opc = AArch64::LD3Rv1d;
6148 selectVectorLoadIntrinsic(Opc, 3,
I);
6151 case Intrinsic::aarch64_neon_ld4: {
6152 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6155 Opc = AArch64::LD4Fourv8b;
6157 Opc = AArch64::LD4Fourv16b;
6159 Opc = AArch64::LD4Fourv4h;
6161 Opc = AArch64::LD4Fourv8h;
6163 Opc = AArch64::LD4Fourv2s;
6165 Opc = AArch64::LD4Fourv4s;
6167 Opc = AArch64::LD4Fourv2d;
6168 else if (Ty ==
S64 || Ty == P0)
6169 Opc = AArch64::LD1Fourv1d;
6172 selectVectorLoadIntrinsic(Opc, 4,
I);
6175 case Intrinsic::aarch64_neon_ld4lane: {
6176 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6179 Opc = AArch64::LD4i8;
6181 Opc = AArch64::LD4i16;
6183 Opc = AArch64::LD4i32;
6186 Opc = AArch64::LD4i64;
6189 if (!selectVectorLoadLaneIntrinsic(Opc, 4,
I))
6193 case Intrinsic::aarch64_neon_ld4r: {
6194 LLT Ty =
MRI.getType(
I.getOperand(0).getReg());
6197 Opc = AArch64::LD4Rv8b;
6199 Opc = AArch64::LD4Rv16b;
6201 Opc = AArch64::LD4Rv4h;
6203 Opc = AArch64::LD4Rv8h;
6205 Opc = AArch64::LD4Rv2s;
6207 Opc = AArch64::LD4Rv4s;
6209 Opc = AArch64::LD4Rv2d;
6210 else if (Ty ==
S64 || Ty == P0)
6211 Opc = AArch64::LD4Rv1d;
6214 selectVectorLoadIntrinsic(Opc, 4,
I);
6217 case Intrinsic::aarch64_neon_st1x2: {
6218 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6221 Opc = AArch64::ST1Twov8b;
6223 Opc = AArch64::ST1Twov16b;
6225 Opc = AArch64::ST1Twov4h;
6227 Opc = AArch64::ST1Twov8h;
6229 Opc = AArch64::ST1Twov2s;
6231 Opc = AArch64::ST1Twov4s;
6233 Opc = AArch64::ST1Twov2d;
6234 else if (Ty ==
S64 || Ty == P0)
6235 Opc = AArch64::ST1Twov1d;
6238 selectVectorStoreIntrinsic(
I, 2, Opc);
6241 case Intrinsic::aarch64_neon_st1x3: {
6242 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6245 Opc = AArch64::ST1Threev8b;
6247 Opc = AArch64::ST1Threev16b;
6249 Opc = AArch64::ST1Threev4h;
6251 Opc = AArch64::ST1Threev8h;
6253 Opc = AArch64::ST1Threev2s;
6255 Opc = AArch64::ST1Threev4s;
6257 Opc = AArch64::ST1Threev2d;
6258 else if (Ty ==
S64 || Ty == P0)
6259 Opc = AArch64::ST1Threev1d;
6262 selectVectorStoreIntrinsic(
I, 3, Opc);
6265 case Intrinsic::aarch64_neon_st1x4: {
6266 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6269 Opc = AArch64::ST1Fourv8b;
6271 Opc = AArch64::ST1Fourv16b;
6273 Opc = AArch64::ST1Fourv4h;
6275 Opc = AArch64::ST1Fourv8h;
6277 Opc = AArch64::ST1Fourv2s;
6279 Opc = AArch64::ST1Fourv4s;
6281 Opc = AArch64::ST1Fourv2d;
6282 else if (Ty ==
S64 || Ty == P0)
6283 Opc = AArch64::ST1Fourv1d;
6286 selectVectorStoreIntrinsic(
I, 4, Opc);
6289 case Intrinsic::aarch64_neon_st2: {
6290 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6293 Opc = AArch64::ST2Twov8b;
6295 Opc = AArch64::ST2Twov16b;
6297 Opc = AArch64::ST2Twov4h;
6299 Opc = AArch64::ST2Twov8h;
6301 Opc = AArch64::ST2Twov2s;
6303 Opc = AArch64::ST2Twov4s;
6305 Opc = AArch64::ST2Twov2d;
6306 else if (Ty ==
S64 || Ty == P0)
6307 Opc = AArch64::ST1Twov1d;
6310 selectVectorStoreIntrinsic(
I, 2, Opc);
6313 case Intrinsic::aarch64_neon_st3: {
6314 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6317 Opc = AArch64::ST3Threev8b;
6319 Opc = AArch64::ST3Threev16b;
6321 Opc = AArch64::ST3Threev4h;
6323 Opc = AArch64::ST3Threev8h;
6325 Opc = AArch64::ST3Threev2s;
6327 Opc = AArch64::ST3Threev4s;
6329 Opc = AArch64::ST3Threev2d;
6330 else if (Ty ==
S64 || Ty == P0)
6331 Opc = AArch64::ST1Threev1d;
6334 selectVectorStoreIntrinsic(
I, 3, Opc);
6337 case Intrinsic::aarch64_neon_st4: {
6338 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6341 Opc = AArch64::ST4Fourv8b;
6343 Opc = AArch64::ST4Fourv16b;
6345 Opc = AArch64::ST4Fourv4h;
6347 Opc = AArch64::ST4Fourv8h;
6349 Opc = AArch64::ST4Fourv2s;
6351 Opc = AArch64::ST4Fourv4s;
6353 Opc = AArch64::ST4Fourv2d;
6354 else if (Ty ==
S64 || Ty == P0)
6355 Opc = AArch64::ST1Fourv1d;
6358 selectVectorStoreIntrinsic(
I, 4, Opc);
6361 case Intrinsic::aarch64_neon_st2lane: {
6362 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6365 Opc = AArch64::ST2i8;
6367 Opc = AArch64::ST2i16;
6369 Opc = AArch64::ST2i32;
6372 Opc = AArch64::ST2i64;
6375 if (!selectVectorStoreLaneIntrinsic(
I, 2, Opc))
6379 case Intrinsic::aarch64_neon_st3lane: {
6380 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6383 Opc = AArch64::ST3i8;
6385 Opc = AArch64::ST3i16;
6387 Opc = AArch64::ST3i32;
6390 Opc = AArch64::ST3i64;
6393 if (!selectVectorStoreLaneIntrinsic(
I, 3, Opc))
6397 case Intrinsic::aarch64_neon_st4lane: {
6398 LLT Ty =
MRI.getType(
I.getOperand(1).getReg());
6401 Opc = AArch64::ST4i8;
6403 Opc = AArch64::ST4i16;
6405 Opc = AArch64::ST4i32;
6408 Opc = AArch64::ST4i64;
6411 if (!selectVectorStoreLaneIntrinsic(
I, 4, Opc))
6415 case Intrinsic::aarch64_mops_memset_tag: {
6428 Register DstDef =
I.getOperand(0).getReg();
6430 Register DstUse =
I.getOperand(2).getReg();
6431 Register ValUse =
I.getOperand(3).getReg();
6432 Register SizeUse =
I.getOperand(4).getReg();
6439 auto Memset = MIB.
buildInstr(AArch64::MOPSMemorySetTaggingPseudo,
6440 {DstDef, SizeDef}, {DstUse, SizeUse, ValUse});
6447 I.eraseFromParent();
6451bool AArch64InstructionSelector::selectIntrinsic(
MachineInstr &
I,
6453 unsigned IntrinID = cast<GIntrinsic>(
I).getIntrinsicID();
6458 case Intrinsic::aarch64_crypto_sha1h: {
6459 Register DstReg =
I.getOperand(0).getReg();
6460 Register SrcReg =
I.getOperand(2).getReg();
6463 if (
MRI.getType(DstReg).getSizeInBits() != 32 ||
6464 MRI.getType(SrcReg).getSizeInBits() != 32)
6469 if (RBI.getRegBank(SrcReg,
MRI,
TRI)->getID() != AArch64::FPRRegBankID) {
6470 SrcReg =
MRI.createVirtualRegister(&AArch64::FPR32RegClass);
6474 RBI.constrainGenericRegister(
I.getOperand(2).getReg(),
6475 AArch64::GPR32RegClass,
MRI);
6478 if (RBI.getRegBank(DstReg,
MRI,
TRI)->getID() != AArch64::FPRRegBankID)
6479 DstReg =
MRI.createVirtualRegister(&AArch64::FPR32RegClass);
6482 auto SHA1Inst = MIB.
buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg});
6486 if (DstReg !=
I.getOperand(0).getReg()) {
6490 RBI.constrainGenericRegister(
I.getOperand(0).getReg(),
6491 AArch64::GPR32RegClass,
MRI);
6494 I.eraseFromParent();
6497 case Intrinsic::frameaddress:
6498 case Intrinsic::returnaddress: {
6502 unsigned Depth =
I.getOperand(2).getImm();
6503 Register DstReg =
I.getOperand(0).getReg();
6504 RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass,
MRI);
6506 if (
Depth == 0 && IntrinID == Intrinsic::returnaddress) {
6507 if (!MFReturnAddr) {
6512 MF,
TII, AArch64::LR, AArch64::GPR64RegClass,
I.getDebugLoc());
6515 if (STI.hasPAuth()) {
6516 MIB.
buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
6523 I.eraseFromParent();
6530 Register NextFrame =
MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
6532 MIB.
buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr}).addImm(0);
6534 FrameAddr = NextFrame;
6537 if (IntrinID == Intrinsic::frameaddress)
6542 if (STI.hasPAuth()) {
6543 Register TmpReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
6544 MIB.
buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
6545 MIB.
buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
6554 I.eraseFromParent();
6557 case Intrinsic::aarch64_neon_tbl2:
6558 SelectTable(
I,
MRI, 2, AArch64::TBLv8i8Two, AArch64::TBLv16i8Two,
false);
6560 case Intrinsic::aarch64_neon_tbl3:
6561 SelectTable(
I,
MRI, 3, AArch64::TBLv8i8Three, AArch64::TBLv16i8Three,
6564 case Intrinsic::aarch64_neon_tbl4:
6565 SelectTable(
I,
MRI, 4, AArch64::TBLv8i8Four, AArch64::TBLv16i8Four,
false);
6567 case Intrinsic::aarch64_neon_tbx2:
6568 SelectTable(
I,
MRI, 2, AArch64::TBXv8i8Two, AArch64::TBXv16i8Two,
true);
6570 case Intrinsic::aarch64_neon_tbx3:
6571 SelectTable(
I,
MRI, 3, AArch64::TBXv8i8Three, AArch64::TBXv16i8Three,
true);
6573 case Intrinsic::aarch64_neon_tbx4:
6574 SelectTable(
I,
MRI, 4, AArch64::TBXv8i8Four, AArch64::TBXv16i8Four,
true);
6576 case Intrinsic::swift_async_context_addr:
6585 I.eraseFromParent();
6620bool AArch64InstructionSelector::selectPtrAuthGlobalValue(
6622 Register DefReg =
I.getOperand(0).getReg();
6625 Register AddrDisc =
I.getOperand(3).getReg();
6626 uint64_t Disc =
I.getOperand(4).getImm();
6634 if (!isUInt<16>(Disc))
6636 "constant discriminator in ptrauth global out of range [0, 0xffff]");
6639 if (!STI.isTargetELF())
6649 if (!
MRI.hasOneDef(OffsetReg))
6652 if (OffsetMI.
getOpcode() != TargetOpcode::G_CONSTANT)
6678 unsigned OpFlags = STI.ClassifyGlobalReference(GV,
TM);
6681 "unsupported non-GOT op flags on ptrauth global reference");
6683 "unsupported non-GOT reference to weak ptrauth global");
6686 bool HasAddrDisc = !AddrDiscVal || *AddrDiscVal != 0;
6693 MIB.
buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X16}, {});
6694 MIB.
buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X17}, {});
6695 MIB.
buildInstr(NeedsGOTLoad ? AArch64::LOADgotPAC : AArch64::MOVaddrPAC)
6698 .
addReg(HasAddrDisc ? AddrDisc : AArch64::XZR)
6702 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
MRI);
6703 I.eraseFromParent();
6715 "unsupported non-zero offset in weak ptrauth global reference");
6720 MIB.
buildInstr(AArch64::LOADauthptrstatic, {DefReg}, {})
6721 .addGlobalAddress(GV,
Offset)
6724 RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
MRI);
6726 I.eraseFromParent();
6730void AArch64InstructionSelector::SelectTable(
MachineInstr &
I,
6732 unsigned NumVec,
unsigned Opc1,
6733 unsigned Opc2,
bool isExt) {
6734 Register DstReg =
I.getOperand(0).getReg();
6739 for (
unsigned i = 0; i < NumVec; i++)
6740 Regs.
push_back(
I.getOperand(i + 2 + isExt).getReg());
6743 Register IdxReg =
I.getOperand(2 + NumVec + isExt).getReg();
6751 I.eraseFromParent();
6755AArch64InstructionSelector::selectShiftA_32(
const MachineOperand &Root)
const {
6757 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
6758 return std::nullopt;
6759 uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
6764AArch64InstructionSelector::selectShiftB_32(
const MachineOperand &Root)
const {
6766 if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
6767 return std::nullopt;
6773AArch64InstructionSelector::selectShiftA_64(
const MachineOperand &Root)
const {
6775 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
6776 return std::nullopt;
6777 uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
6782AArch64InstructionSelector::selectShiftB_64(
const MachineOperand &Root)
const {
6784 if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
6785 return std::nullopt;
6796AArch64InstructionSelector::select12BitValueWithLeftShift(
6799 if (Immed >> 12 == 0) {
6801 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
6803 Immed = Immed >> 12;
6805 return std::nullopt;
6818AArch64InstructionSelector::selectArithImmed(
MachineOperand &Root)
const {
6825 if (MaybeImmed == std::nullopt)
6826 return std::nullopt;
6827 return select12BitValueWithLeftShift(*MaybeImmed);
6833AArch64InstructionSelector::selectNegArithImmed(
MachineOperand &Root)
const {
6837 return std::nullopt;
6839 if (MaybeImmed == std::nullopt)
6840 return std::nullopt;
6847 return std::nullopt;
6852 if (
MRI.getType(Root.
getReg()).getSizeInBits() == 32)
6855 Immed = ~Immed + 1ULL;
6857 if (Immed & 0xFFFFFFFFFF000000ULL)
6858 return std::nullopt;
6860 Immed &= 0xFFFFFFULL;
6861 return select12BitValueWithLeftShift(Immed);
6867bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
6871 if (
MRI.hasOneNonDBGUse(DefReg) ||
6872 MI.getParent()->getParent()->getFunction().hasOptSize())
6881 return all_of(
MRI.use_nodbg_instructions(DefReg),
6897AArch64InstructionSelector::selectExtendedSHL(
6899 unsigned SizeInBytes,
bool WantsExt)
const {
6900 assert(
Base.isReg() &&
"Expected base to be a register operand");
6901 assert(
Offset.isReg() &&
"Expected offset to be a register operand");
6906 unsigned OffsetOpc = OffsetInst->
getOpcode();
6907 bool LookedThroughZExt =
false;
6908 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
6910 if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
6911 return std::nullopt;
6915 LookedThroughZExt =
true;
6917 if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
6918 return std::nullopt;
6921 int64_t LegalShiftVal =
Log2_32(SizeInBytes);
6922 if (LegalShiftVal == 0)
6923 return std::nullopt;
6924 if (!isWorthFoldingIntoExtendedReg(*OffsetInst,
MRI))
6925 return std::nullopt;
6936 if (OffsetOpc == TargetOpcode::G_SHL)
6937 return std::nullopt;
6943 return std::nullopt;
6948 int64_t ImmVal = ValAndVReg->Value.getSExtValue();
6952 if (OffsetOpc == TargetOpcode::G_MUL) {
6953 if (!llvm::has_single_bit<uint32_t>(ImmVal))
6954 return std::nullopt;
6960 if ((ImmVal & 0x7) != ImmVal)
6961 return std::nullopt;
6965 if (ImmVal != LegalShiftVal)
6966 return std::nullopt;
6968 unsigned SignExtend = 0;
6972 if (!LookedThroughZExt) {
6974 auto Ext = getExtendTypeForInst(*ExtInst,
MRI,
true);
6976 return std::nullopt;
6981 return std::nullopt;
6987 OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
6997 MIB.addImm(SignExtend);
7011AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
7014 return std::nullopt;
7031 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd,
MRI))
7032 return std::nullopt;
7038 return selectExtendedSHL(Root, PtrAdd->
getOperand(1),
7052AArch64InstructionSelector::selectAddrModeRegisterOffset(
7058 if (Gep->
getOpcode() != TargetOpcode::G_PTR_ADD)
7059 return std::nullopt;
7065 return std::nullopt;
7085AArch64InstructionSelector::selectAddrModeXRO(
MachineOperand &Root,
7086 unsigned SizeInBytes)
const {
7089 return std::nullopt;
7093 return std::nullopt;
7111 unsigned Scale =
Log2_32(SizeInBytes);
7112 int64_t ImmOff = ValAndVReg->Value.getSExtValue();
7116 if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
7117 ImmOff < (0x1000 << Scale))
7118 return std::nullopt;
7123 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
7127 if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
7133 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
7134 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
7139 return std::nullopt;
7143 auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
7149 return selectAddrModeRegisterOffset(Root);
7159AArch64InstructionSelector::selectAddrModeWRO(
MachineOperand &Root,
7160 unsigned SizeInBytes)
const {
7165 if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd,
MRI))
7166 return std::nullopt;
7187 auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->
getOperand(0),
7196 if (!isWorthFoldingIntoExtendedReg(*OffsetInst,
MRI))
7197 return std::nullopt;
7201 getExtendTypeForInst(*OffsetInst,
MRI,
true);
7203 return std::nullopt;
7208 AArch64::GPR32RegClass, MIB);
7215 MIB.addImm(SignExtend);
7226AArch64InstructionSelector::selectAddrModeUnscaled(
MachineOperand &Root,
7227 unsigned Size)
const {
7232 return std::nullopt;
7234 if (!isBaseWithConstantOffset(Root,
MRI))
7235 return std::nullopt;
7240 if (!OffImm.
isReg())
7241 return std::nullopt;
7243 if (
RHS->getOpcode() != TargetOpcode::G_CONSTANT)
7244 return std::nullopt;
7248 return std::nullopt;
7251 if (RHSC >= -256 && RHSC < 256) {
7258 return std::nullopt;
7262AArch64InstructionSelector::tryFoldAddLowIntoImm(
MachineInstr &RootDef,
7265 if (RootDef.
getOpcode() != AArch64::G_ADD_LOW)
7266 return std::nullopt;
7269 return std::nullopt;
7274 return std::nullopt;
7278 return std::nullopt;
7282 return std::nullopt;
7284 unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.
getTarget());
7289 MIB.addGlobalAddress(GV,
Offset,
7299AArch64InstructionSelector::selectAddrModeIndexed(
MachineOperand &Root,
7300 unsigned Size)
const {
7305 return std::nullopt;
7308 if (RootDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7318 auto OpFns = tryFoldAddLowIntoImm(*RootDef,
Size,
MRI);
7323 if (isBaseWithConstantOffset(Root,
MRI)) {
7331 if ((RHSC & (
Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
7332 if (LHSDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX)
7347 if (selectAddrModeUnscaled(Root,
Size))
7348 return std::nullopt;
7359 switch (
MI.getOpcode()) {
7362 case TargetOpcode::G_SHL:
7364 case TargetOpcode::G_LSHR:
7366 case TargetOpcode::G_ASHR:
7368 case TargetOpcode::G_ROTR:
7376AArch64InstructionSelector::selectShiftedRegister(
MachineOperand &Root,
7377 bool AllowROR)
const {
7379 return std::nullopt;
7388 return std::nullopt;
7390 return std::nullopt;
7391 if (!isWorthFoldingIntoExtendedReg(*ShiftInst,
MRI))
7392 return std::nullopt;
7398 return std::nullopt;
7405 unsigned NumBits =
MRI.getType(ShiftReg).getSizeInBits();
7406 unsigned Val = *Immed & (NumBits - 1);
7415 unsigned Opc =
MI.getOpcode();
7418 if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
7420 if (Opc == TargetOpcode::G_SEXT)
7421 Size =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
7423 Size =
MI.getOperand(2).getImm();
7424 assert(
Size != 64 &&
"Extend from 64 bits?");
7437 if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
7438 unsigned Size =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
7439 assert(
Size != 64 &&
"Extend from 64 bits?");
7454 if (Opc != TargetOpcode::G_AND)
7473Register AArch64InstructionSelector::moveScalarRegClass(
7476 auto Ty =
MRI.getType(Reg);
7485 return Copy.getReg(0);
7491AArch64InstructionSelector::selectArithExtendedRegister(
7494 return std::nullopt;
7503 return std::nullopt;
7505 if (!isWorthFoldingIntoExtendedReg(*RootDef,
MRI))
7506 return std::nullopt;
7509 if (RootDef->
getOpcode() == TargetOpcode::G_SHL) {
7514 return std::nullopt;
7515 ShiftVal = *MaybeShiftVal;
7517 return std::nullopt;
7522 return std::nullopt;
7523 Ext = getExtendTypeForInst(*ExtDef,
MRI);
7525 return std::nullopt;
7529 Ext = getExtendTypeForInst(*RootDef,
MRI);
7531 return std::nullopt;
7540 if (isDef32(*ExtInst))
7541 return std::nullopt;
7548 ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);
7552 MIB.addImm(getArithExtendImm(Ext, ShiftVal));
7557AArch64InstructionSelector::selectExtractHigh(
MachineOperand &Root)
const {
7559 return std::nullopt;
7564 while (Extract && Extract->MI->
getOpcode() == TargetOpcode::G_BITCAST &&
7565 STI.isLittleEndian())
7569 return std::nullopt;
7571 if (Extract->MI->
getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
7577 if (Extract->MI->
getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) {
7582 LaneIdx->Value.getSExtValue() == 1) {
7588 return std::nullopt;
7595 assert(
MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7596 "Expected G_CONSTANT");
7597 std::optional<int64_t> CstVal =
7599 assert(CstVal &&
"Expected constant value");
7603void AArch64InstructionSelector::renderLogicalImm32(
7605 assert(
I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7606 "Expected G_CONSTANT");
7607 uint64_t CstVal =
I.getOperand(1).getCImm()->getZExtValue();
7612void AArch64InstructionSelector::renderLogicalImm64(
7614 assert(
I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7615 "Expected G_CONSTANT");
7616 uint64_t CstVal =
I.getOperand(1).getCImm()->getZExtValue();
7624 assert(
MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
7625 "Expected G_UBSANTRAP");
7626 MIB.
addImm(
MI.getOperand(0).getImm() | (
'U' << 8));
7632 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
7633 "Expected G_FCONSTANT");
7641 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
7642 "Expected G_FCONSTANT");
7650 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
7651 "Expected G_FCONSTANT");
7656void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
7658 assert(
MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
7659 "Expected G_FCONSTANT");
7667bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
7669 if (!
MI.mayLoadOrStore())
7672 "Expected load/store to have only one mem op!");
7673 return (*
MI.memoperands_begin())->getSize() == NumBytes;
7676bool AArch64InstructionSelector::isDef32(
const MachineInstr &
MI)
const {
7678 if (
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits() != 32)
7685 switch (
MI.getOpcode()) {
7688 case TargetOpcode::COPY:
7689 case TargetOpcode::G_BITCAST:
7690 case TargetOpcode::G_TRUNC:
7691 case TargetOpcode::G_PHI:
7701 assert(
MI.getOpcode() == TargetOpcode::G_PHI &&
"Expected a G_PHI");
7704 assert(DstRB &&
"Expected PHI dst to have regbank assigned");
7715 auto *OpDef =
MRI.getVRegDef(OpReg);
7716 const LLT &Ty =
MRI.getType(OpReg);
7722 if (InsertPt != OpDefBB.
end() && InsertPt->isPHI())
7726 MRI.setRegBank(Copy.getReg(0), *DstRB);
7727 MO.setReg(Copy.getReg(0));
7736 for (
auto &BB : MF) {
7737 for (
auto &
MI : BB) {
7738 if (
MI.getOpcode() == TargetOpcode::G_PHI)
7743 for (
auto *
MI : Phis) {
7765 bool HasGPROp =
false, HasFPROp =
false;
7769 const LLT &Ty =
MRI.getType(MO.getReg());
7779 if (RB->
getID() == AArch64::GPRRegBankID)
7785 if (HasGPROp && HasFPROp)
7795 return new AArch64InstructionSelector(
TM, Subtarget, RBI);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool isPreferredADD(int64_t ImmOff)
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A))...
static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool isCMN(SDValue Op, ISD::CondCode CC)
static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy)
#define GET_GLOBALISEL_PREDICATES_INIT
static std::pair< const TargetRegisterClass *, const TargetRegisterClass * > getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Helper function to get the source and destination register classes for a copy.
#define GET_GLOBALISEL_TEMPORARIES_INIT
static void changeFPCCToANDAArch64CC(CmpInst::Predicate CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Convert an IR fp condition code to an AArch64 CC.
static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert, MachineRegisterInfo &MRI)
Return a register which can be used as a bit to test in a TB(N)Z.
static unsigned getMinSizeForRegBank(const RegisterBank &RB)
Returns the minimum size the given register bank can hold.
static std::optional< int64_t > getVectorShiftImm(Register Reg, MachineRegisterInfo &MRI)
Returns the element immediate value of a vector shift operand if found.
static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the G_LOAD or G_STORE operation GenericOpc, appropriate for the (value)...
static const TargetRegisterClass * getMinClassForRegBank(const RegisterBank &RB, TypeSize SizeInBits, bool GetAllRegSet=false)
Given a register bank, and size in bits, return the smallest register class that can represent that c...
static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P)
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV),...
static bool getSubRegForClass(const TargetRegisterClass *RC, const TargetRegisterInfo &TRI, unsigned &SubReg)
Returns the correct subregister to use for a given register class.
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI, const RegisterBankInfo &RBI, Register SrcReg, const TargetRegisterClass *To, unsigned SubReg)
Helper function for selectCopy.
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI, const AArch64RegisterBankInfo &RBI)
static bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI, const RegisterBankInfo &RBI)
static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type)
static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI)
Given a shift instruction, return the correct shift type for that instruction.
static bool unsupportedBinOp(const MachineInstr &I, const AArch64RegisterBankInfo &RBI, const MachineRegisterInfo &MRI, const AArch64RegisterInfo &TRI)
Check whether I is a currently unsupported binary operation:
static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, MachineRegisterInfo &MRI, unsigned Depth=0)
Returns true if Val is a tree of AND/OR/CMP operations that can be expressed as a conjunction.
static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg, const unsigned EltSize)
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static std::optional< uint64_t > getImmedFromMO(const MachineOperand &Root)
static std::pair< unsigned, unsigned > getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize)
Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given size and RB.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
static std::optional< int64_t > getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI)
Matches and returns the shift immediate value for a SHL instruction given a shift operand.
static void changeFPCCToORAArch64CC(CmpInst::Predicate CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
This file declares the targeting of the RegisterBankInfo class for AArch64.
static bool isStore(int Opcode)
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file contains constants used for implementing Dwarf debug support.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder)
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static constexpr int Concat[]
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
int getVarArgsStackIndex() const
int getVarArgsGPRIndex() const
unsigned getVarArgsGPRSize() const
This class provides the information for the target register banks.
bool isCallingConvWin64(CallingConv::ID CC) const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
uint64_t getZExtValue() const
Get zero extended value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
bool isIntPredicate() const
static Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
bool isNegative() const
Return true if the sign bit is set.
bool isZero() const
Return true if the value is positive or negative zero.
This is the shared class of boolean and integer constants.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers,...
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
Represents indexed stores.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSize() const
Returns the size in bytes of the memory access.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
Register getCondReg() const
Register getFalseReg() const
Register getTrueReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
bool hasExternalWeakLinkage() const
bool isEquality() const
Return true if this predicate is either EQ or NE.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
constexpr LLT multiplyElements(int Factor) const
Produce a vector type that is Factor times bigger, preserving the element type.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
This is an important class for using LLVM in a threaded context.
TypeSize getValue() const
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setAdjustsStack(bool V)
void setFrameAddressIsTaken(bool T)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineIRBuilderState & getState()
Getter for the State.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
void setState(const MachineIRBuilderState &NewState)
Setter for the State.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addBlockAddress(const BlockAddress *BA, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MOLoad
The memory access reads data.
AtomicOrdering getSuccessOrdering() const
Return the atomic ordering requirements for this memory operation.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
static MachineOperand CreateBA(const BlockAddress *BA, int64_t Offset, unsigned TargetFlags=0)
const ConstantFP * getFPImm() const
unsigned getPredicate() const
int64_t getOffset() const
Return the offset from the symbol in this operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
T get() const
Returns the value of the specified pointer type.
T dyn_cast() const
Returns the current pointer if it is of the specified pointer type, otherwise returns null.
Analysis providing profile information.
Holds all the information related to register banks.
static const TargetRegisterClass * constrainGenericRegister(Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI)
Constrain the (possibly generic) virtual register Reg to RC.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
TypeSize getSizeInBits(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const
Get the size in bits of Reg.
This class implements the register bank concept.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
This class represents the LLVM 'select' instruction.
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetInstrInfo - Interface to description of machine instruction set.
CodeModel::Model getCodeModel() const
Returns the code model.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetLowering * getTargetLowering() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static IntegerType * getInt8Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
void changeFCMPPredToAArch64CC(const CmpInst::Predicate P, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Find the AArch64 condition codes necessary to represent P for a scalar floating point comparison.
std::optional< int64_t > getAArch64VectorSplatScalar(const MachineInstr &MI, const MachineRegisterInfo &MRI)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint8_t encodeAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType9(uint64_t Imm)
static bool isAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType5(uint64_t Imm)
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType10(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType9(uint64_t Imm)
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static bool isAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType5(uint64_t Imm)
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static bool isAdvSIMDModImmType10(uint64_t Imm)
static int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType8(uint64_t Imm)
static bool isAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType11(uint64_t Imm)
static bool isAdvSIMDModImmType11(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType6(uint64_t Imm)
static bool isAdvSIMDModImmType8(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType4(uint64_t Imm)
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static bool isAdvSIMDModImmType6(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType1(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType1(uint64_t Imm)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(int64_t RequestedValue)
Matches a constant equal to RequestedValue.
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
NodeAddr< InstrNode * > Instr
This is an optimization pass for GlobalISel generic memory operations.
Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
bool shouldOptForSize(const MachineBasicBlock &MBB, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
Returns true if the given block should be optimized for size.
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
std::optional< ValueAndVReg > getAnyConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT or G_FCONST...
AtomicOrdering
Atomic ordering for LLVM's memory model.
InstructionSelector * createAArch64InstructionSelector(const AArch64TargetMachine &, AArch64Subtarget &, AArch64RegisterBankInfo &)
DWARFExpression::Operation Op
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Class which stores all the state required in a MachineIRBuilder.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.