62#define GET_INSTRINFO_CTOR_DTOR
63#include "AArch64GenInstrInfo.inc"
67 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
71 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
75 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
79 cl::desc(
"Restrict range of B instructions (DEBUG)"));
84 RI(STI.getTargetTriple()), Subtarget(STI) {}
95 auto Op =
MI.getOpcode();
96 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
97 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
101 if (
MI.isMetaInstruction())
106 unsigned NumBytes = 0;
110 NumBytes =
Desc.getSize() ?
Desc.getSize() : 4;
113 if (!MFI->shouldSignReturnAddress(MF))
125 switch (
Desc.getOpcode()) {
128 return Desc.getSize();
135 case TargetOpcode::STACKMAP:
138 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
140 case TargetOpcode::PATCHPOINT:
143 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
145 case TargetOpcode::STATEPOINT:
147 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
152 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
157 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
159 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
160 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
164 case TargetOpcode::PATCHABLE_EVENT_CALL:
170 NumBytes =
MI.getOperand(1).getImm();
172 case TargetOpcode::BUNDLE:
173 NumBytes = getInstBundleLength(
MI);
180unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
184 while (++
I != E &&
I->isInsideBundle()) {
185 assert(!
I->isBundle() &&
"No nested bundle!");
244 int64_t BrOffset)
const {
246 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
247 "over conditional branch expansion");
248 return isIntN(Bits, BrOffset / 4);
253 switch (
MI.getOpcode()) {
257 return MI.getOperand(0).getMBB();
262 return MI.getOperand(2).getMBB();
268 return MI.getOperand(1).getMBB();
278 assert(RS &&
"RegScavenger required for long branching");
280 "new block should be inserted for expanding unconditional branch");
283 "restore block should be inserted for restoring clobbered registers");
288 if (!isInt<33>(BrOffset))
290 "Branch offsets outside of the signed 33-bit range not supported");
304 constexpr Register Reg = AArch64::X16;
306 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
314 if (Scavenged != AArch64::NoRegister &&
316 buildIndirectBranch(Scavenged, NewDestBB);
326 "Unable to insert indirect branch inside function that has red zone");
349 bool AllowModify)
const {
356 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
357 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
361 if (!isUnpredicatedTerminator(*
I))
368 unsigned LastOpc = LastInst->
getOpcode();
369 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
384 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
391 LastInst = SecondLastInst;
393 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
398 SecondLastInst = &*
I;
399 SecondLastOpc = SecondLastInst->
getOpcode();
410 LastInst = SecondLastInst;
412 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
414 "unreachable unconditional branches removed above");
423 SecondLastInst = &*
I;
424 SecondLastOpc = SecondLastInst->
getOpcode();
428 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
444 I->eraseFromParent();
453 I->eraseFromParent();
462 MachineBranchPredicate &MBP,
463 bool AllowModify)
const {
473 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
474 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
478 if (!isUnpredicatedTerminator(*
I))
483 unsigned LastOpc = LastInst->
getOpcode();
498 assert(MBP.TrueDest &&
"expected!");
501 MBP.ConditionDef =
nullptr;
502 MBP.SingleUseCondition =
false;
506 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
507 : MachineBranchPredicate::PRED_EQ;
513 if (
Cond[0].getImm() != -1) {
519 switch (
Cond[1].getImm()) {
523 Cond[1].setImm(AArch64::CBNZW);
526 Cond[1].setImm(AArch64::CBZW);
529 Cond[1].setImm(AArch64::CBNZX);
532 Cond[1].setImm(AArch64::CBZX);
535 Cond[1].setImm(AArch64::TBNZW);
538 Cond[1].setImm(AArch64::TBZW);
541 Cond[1].setImm(AArch64::TBNZX);
544 Cond[1].setImm(AArch64::TBZX);
553 int *BytesRemoved)
const {
563 I->eraseFromParent();
580 I->eraseFromParent();
587void AArch64InstrInfo::instantiateCondBranch(
590 if (
Cond[0].getImm() != -1) {
608 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
647 unsigned *NewVReg =
nullptr) {
652 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
655 unsigned SrcOpNum = 0;
657 case AArch64::ADDSXri:
658 case AArch64::ADDSWri:
665 case AArch64::ADDXri:
666 case AArch64::ADDWri:
672 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
675 case AArch64::ORNXrr:
676 case AArch64::ORNWrr: {
679 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
682 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
686 case AArch64::SUBSXrr:
687 case AArch64::SUBSWrr:
694 case AArch64::SUBXrr:
695 case AArch64::SUBWrr: {
698 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
701 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
707 assert(Opc && SrcOpNum &&
"Missing parameters");
719 int &FalseCycles)
const {
723 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
730 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
734 unsigned ExtraCondLat =
Cond.size() != 1;
738 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
739 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
741 CondCycles = 1 + ExtraCondLat;
742 TrueCycles = FalseCycles = 1;
752 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
753 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
754 CondCycles = 5 + ExtraCondLat;
755 TrueCycles = FalseCycles = 2;
772 switch (
Cond.size()) {
781 switch (
Cond[1].getImm()) {
804 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
810 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
820 switch (
Cond[1].getImm()) {
833 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
849 bool TryFold =
false;
850 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
851 RC = &AArch64::GPR64RegClass;
852 Opc = AArch64::CSELXr;
854 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
855 RC = &AArch64::GPR32RegClass;
856 Opc = AArch64::CSELWr;
858 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
859 RC = &AArch64::FPR64RegClass;
860 Opc = AArch64::FCSELDrrr;
861 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
862 RC = &AArch64::FPR32RegClass;
863 Opc = AArch64::FCSELSrrr;
865 assert(RC &&
"Unsupported regclass");
869 unsigned NewVReg = 0;
884 MRI.clearKillFlags(NewVReg);
889 MRI.constrainRegClass(TrueReg, RC);
890 MRI.constrainRegClass(FalseReg, RC);
905 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
910 return Is.
size() <= 2;
916 if (Subtarget.hasExynosCheapAsMoveHandling()) {
917 if (isExynosCheapAsMove(
MI))
919 return MI.isAsCheapAsAMove();
922 switch (
MI.getOpcode()) {
924 return MI.isAsCheapAsAMove();
926 case AArch64::ADDWrs:
927 case AArch64::ADDXrs:
928 case AArch64::SUBWrs:
929 case AArch64::SUBXrs:
930 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
935 case AArch64::MOVi32imm:
937 case AArch64::MOVi64imm:
943 switch (
MI.getOpcode()) {
947 case AArch64::ADDWrs:
948 case AArch64::ADDXrs:
949 case AArch64::ADDSWrs:
950 case AArch64::ADDSXrs: {
951 unsigned Imm =
MI.getOperand(3).getImm();
958 case AArch64::ADDWrx:
959 case AArch64::ADDXrx:
960 case AArch64::ADDXrx64:
961 case AArch64::ADDSWrx:
962 case AArch64::ADDSXrx:
963 case AArch64::ADDSXrx64: {
964 unsigned Imm =
MI.getOperand(3).getImm();
976 case AArch64::SUBWrs:
977 case AArch64::SUBSWrs: {
978 unsigned Imm =
MI.getOperand(3).getImm();
980 return ShiftVal == 0 ||
984 case AArch64::SUBXrs:
985 case AArch64::SUBSXrs: {
986 unsigned Imm =
MI.getOperand(3).getImm();
988 return ShiftVal == 0 ||
992 case AArch64::SUBWrx:
993 case AArch64::SUBXrx:
994 case AArch64::SUBXrx64:
995 case AArch64::SUBSWrx:
996 case AArch64::SUBSXrx:
997 case AArch64::SUBSXrx64: {
998 unsigned Imm =
MI.getOperand(3).getImm();
1010 case AArch64::LDRBBroW:
1011 case AArch64::LDRBBroX:
1012 case AArch64::LDRBroW:
1013 case AArch64::LDRBroX:
1014 case AArch64::LDRDroW:
1015 case AArch64::LDRDroX:
1016 case AArch64::LDRHHroW:
1017 case AArch64::LDRHHroX:
1018 case AArch64::LDRHroW:
1019 case AArch64::LDRHroX:
1020 case AArch64::LDRQroW:
1021 case AArch64::LDRQroX:
1022 case AArch64::LDRSBWroW:
1023 case AArch64::LDRSBWroX:
1024 case AArch64::LDRSBXroW:
1025 case AArch64::LDRSBXroX:
1026 case AArch64::LDRSHWroW:
1027 case AArch64::LDRSHWroX:
1028 case AArch64::LDRSHXroW:
1029 case AArch64::LDRSHXroX:
1030 case AArch64::LDRSWroW:
1031 case AArch64::LDRSWroX:
1032 case AArch64::LDRSroW:
1033 case AArch64::LDRSroX:
1034 case AArch64::LDRWroW:
1035 case AArch64::LDRWroX:
1036 case AArch64::LDRXroW:
1037 case AArch64::LDRXroX:
1038 case AArch64::PRFMroW:
1039 case AArch64::PRFMroX:
1040 case AArch64::STRBBroW:
1041 case AArch64::STRBBroX:
1042 case AArch64::STRBroW:
1043 case AArch64::STRBroX:
1044 case AArch64::STRDroW:
1045 case AArch64::STRDroX:
1046 case AArch64::STRHHroW:
1047 case AArch64::STRHHroX:
1048 case AArch64::STRHroW:
1049 case AArch64::STRHroX:
1050 case AArch64::STRQroW:
1051 case AArch64::STRQroX:
1052 case AArch64::STRSroW:
1053 case AArch64::STRSroX:
1054 case AArch64::STRWroW:
1055 case AArch64::STRWroX:
1056 case AArch64::STRXroW:
1057 case AArch64::STRXroX: {
1058 unsigned IsSigned =
MI.getOperand(3).getImm();
1065 unsigned Opc =
MI.getOpcode();
1069 case AArch64::SEH_StackAlloc:
1070 case AArch64::SEH_SaveFPLR:
1071 case AArch64::SEH_SaveFPLR_X:
1072 case AArch64::SEH_SaveReg:
1073 case AArch64::SEH_SaveReg_X:
1074 case AArch64::SEH_SaveRegP:
1075 case AArch64::SEH_SaveRegP_X:
1076 case AArch64::SEH_SaveFReg:
1077 case AArch64::SEH_SaveFReg_X:
1078 case AArch64::SEH_SaveFRegP:
1079 case AArch64::SEH_SaveFRegP_X:
1080 case AArch64::SEH_SetFP:
1081 case AArch64::SEH_AddFP:
1082 case AArch64::SEH_Nop:
1083 case AArch64::SEH_PrologEnd:
1084 case AArch64::SEH_EpilogStart:
1085 case AArch64::SEH_EpilogEnd:
1086 case AArch64::SEH_PACSignLR:
1087 case AArch64::SEH_SaveAnyRegQP:
1088 case AArch64::SEH_SaveAnyRegQPX:
1095 unsigned &SubIdx)
const {
1096 switch (
MI.getOpcode()) {
1099 case AArch64::SBFMXri:
1100 case AArch64::UBFMXri:
1103 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1106 SrcReg =
MI.getOperand(1).getReg();
1107 DstReg =
MI.getOperand(0).getReg();
1108 SubIdx = AArch64::sub_32;
1117 int64_t OffsetA = 0, OffsetB = 0;
1118 TypeSize WidthA(0,
false), WidthB(0,
false);
1119 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1140 OffsetAIsScalable == OffsetBIsScalable) {
1141 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1142 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1143 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1144 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1162 switch (
MI.getOpcode()) {
1165 if (
MI.getOperand(0).getImm() == 0x14)
1172 case AArch64::MSRpstatesvcrImm1:
1179 auto Next = std::next(
MI.getIterator());
1180 return Next !=
MBB->
end() && Next->isCFIInstruction();
1187 Register &SrcReg2, int64_t &CmpMask,
1188 int64_t &CmpValue)
const {
1191 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1192 if (!
MI.getOperand(1).isReg())
1195 switch (
MI.getOpcode()) {
1198 case AArch64::PTEST_PP:
1199 case AArch64::PTEST_PP_ANY:
1200 SrcReg =
MI.getOperand(0).getReg();
1201 SrcReg2 =
MI.getOperand(1).getReg();
1206 case AArch64::SUBSWrr:
1207 case AArch64::SUBSWrs:
1208 case AArch64::SUBSWrx:
1209 case AArch64::SUBSXrr:
1210 case AArch64::SUBSXrs:
1211 case AArch64::SUBSXrx:
1212 case AArch64::ADDSWrr:
1213 case AArch64::ADDSWrs:
1214 case AArch64::ADDSWrx:
1215 case AArch64::ADDSXrr:
1216 case AArch64::ADDSXrs:
1217 case AArch64::ADDSXrx:
1219 SrcReg =
MI.getOperand(1).getReg();
1220 SrcReg2 =
MI.getOperand(2).getReg();
1224 case AArch64::SUBSWri:
1225 case AArch64::ADDSWri:
1226 case AArch64::SUBSXri:
1227 case AArch64::ADDSXri:
1228 SrcReg =
MI.getOperand(1).getReg();
1231 CmpValue =
MI.getOperand(2).getImm();
1233 case AArch64::ANDSWri:
1234 case AArch64::ANDSXri:
1237 SrcReg =
MI.getOperand(1).getReg();
1241 MI.getOperand(2).getImm(),
1242 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1251 assert(
MBB &&
"Can't get MachineBasicBlock here");
1253 assert(MF &&
"Can't get MachineFunction here");
1258 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1262 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1265 if (!OpRegCstraints)
1273 "Operand has register constraints without being a register!");
1276 if (Reg.isPhysical()) {
1277 if (!OpRegCstraints->
contains(Reg))
1280 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1293 bool MIDefinesZeroReg =
false;
1294 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1295 MI.definesRegister(AArch64::XZR,
nullptr))
1296 MIDefinesZeroReg =
true;
1298 switch (
MI.getOpcode()) {
1300 return MI.getOpcode();
1301 case AArch64::ADDSWrr:
1302 return AArch64::ADDWrr;
1303 case AArch64::ADDSWri:
1304 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1305 case AArch64::ADDSWrs:
1306 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1307 case AArch64::ADDSWrx:
1308 return AArch64::ADDWrx;
1309 case AArch64::ADDSXrr:
1310 return AArch64::ADDXrr;
1311 case AArch64::ADDSXri:
1312 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1313 case AArch64::ADDSXrs:
1314 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1315 case AArch64::ADDSXrx:
1316 return AArch64::ADDXrx;
1317 case AArch64::SUBSWrr:
1318 return AArch64::SUBWrr;
1319 case AArch64::SUBSWri:
1320 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1321 case AArch64::SUBSWrs:
1322 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1323 case AArch64::SUBSWrx:
1324 return AArch64::SUBWrx;
1325 case AArch64::SUBSXrr:
1326 return AArch64::SUBXrr;
1327 case AArch64::SUBSXri:
1328 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1329 case AArch64::SUBSXrs:
1330 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1331 case AArch64::SUBSXrx:
1332 return AArch64::SUBXrx;
1347 if (To == To->getParent()->begin())
1352 if (To->getParent() !=
From->getParent())
1364 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1365 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1371std::optional<unsigned>
1375 unsigned MaskOpcode =
Mask->getOpcode();
1376 unsigned PredOpcode = Pred->
getOpcode();
1380 if (PredIsWhileLike) {
1384 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1398 if (PredIsPTestLike) {
1403 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1413 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1442 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1450 switch (PredOpcode) {
1451 case AArch64::AND_PPzPP:
1452 case AArch64::BIC_PPzPP:
1453 case AArch64::EOR_PPzPP:
1454 case AArch64::NAND_PPzPP:
1455 case AArch64::NOR_PPzPP:
1456 case AArch64::ORN_PPzPP:
1457 case AArch64::ORR_PPzPP:
1458 case AArch64::BRKA_PPzP:
1459 case AArch64::BRKPA_PPzPP:
1460 case AArch64::BRKB_PPzP:
1461 case AArch64::BRKPB_PPzPP:
1462 case AArch64::RDFFR_PPz: {
1466 if (Mask != PredMask)
1470 case AArch64::BRKN_PPzP: {
1474 if ((MaskOpcode != AArch64::PTRUE_B) ||
1475 (
Mask->getOperand(1).getImm() != 31))
1479 case AArch64::PTRUE_B:
1492bool AArch64InstrInfo::optimizePTestInstr(
1493 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1495 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1496 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1497 unsigned PredOpcode = Pred->
getOpcode();
1498 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1514 if (*NewOp != PredOpcode) {
1525 for (; i !=
e; ++i) {
1556 if (DeadNZCVIdx != -1) {
1575 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1576 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1577 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1586 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1588 return (CmpValue == 0 || CmpValue == 1) &&
1589 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1597 switch (Instr.getOpcode()) {
1599 return AArch64::INSTRUCTION_LIST_END;
1601 case AArch64::ADDSWrr:
1602 case AArch64::ADDSWri:
1603 case AArch64::ADDSXrr:
1604 case AArch64::ADDSXri:
1605 case AArch64::SUBSWrr:
1606 case AArch64::SUBSWri:
1607 case AArch64::SUBSXrr:
1608 case AArch64::SUBSXri:
1609 return Instr.getOpcode();
1611 case AArch64::ADDWrr:
1612 return AArch64::ADDSWrr;
1613 case AArch64::ADDWri:
1614 return AArch64::ADDSWri;
1615 case AArch64::ADDXrr:
1616 return AArch64::ADDSXrr;
1617 case AArch64::ADDXri:
1618 return AArch64::ADDSXri;
1619 case AArch64::ADCWr:
1620 return AArch64::ADCSWr;
1621 case AArch64::ADCXr:
1622 return AArch64::ADCSXr;
1623 case AArch64::SUBWrr:
1624 return AArch64::SUBSWrr;
1625 case AArch64::SUBWri:
1626 return AArch64::SUBSWri;
1627 case AArch64::SUBXrr:
1628 return AArch64::SUBSXrr;
1629 case AArch64::SUBXri:
1630 return AArch64::SUBSXri;
1631 case AArch64::SBCWr:
1632 return AArch64::SBCSWr;
1633 case AArch64::SBCXr:
1634 return AArch64::SBCSXr;
1635 case AArch64::ANDWri:
1636 return AArch64::ANDSWri;
1637 case AArch64::ANDXri:
1638 return AArch64::ANDSXri;
1645 if (BB->isLiveIn(AArch64::NZCV))
1654 switch (Instr.getOpcode()) {
1658 case AArch64::Bcc: {
1659 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1664 case AArch64::CSINVWr:
1665 case AArch64::CSINVXr:
1666 case AArch64::CSINCWr:
1667 case AArch64::CSINCXr:
1668 case AArch64::CSELWr:
1669 case AArch64::CSELXr:
1670 case AArch64::CSNEGWr:
1671 case AArch64::CSNEGXr:
1672 case AArch64::FCSELSrrr:
1673 case AArch64::FCSELDrrr: {
1674 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1687 Instr.getOperand(CCIdx).getImm())
1740std::optional<UsedNZCV>
1745 if (
MI.getParent() != CmpParent)
1746 return std::nullopt;
1749 return std::nullopt;
1754 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1757 return std::nullopt;
1762 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1765 return NZCVUsedAfterCmp;
1769 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1773 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1795 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1801 "Caller guarantees that CmpInstr compares with constant 0");
1804 if (!NZVCUsed || NZVCUsed->C)
1826bool AArch64InstrInfo::substituteCmpToZero(
1837 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1844 MI->setDesc(
get(NewOpc));
1849 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1861 assert((CmpValue == 0 || CmpValue == 1) &&
1862 "Only comparisons to 0 or 1 considered for removal!");
1865 unsigned MIOpc =
MI.getOpcode();
1866 if (MIOpc == AArch64::CSINCWr) {
1867 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1868 MI.getOperand(2).getReg() != AArch64::WZR)
1870 }
else if (MIOpc == AArch64::CSINCXr) {
1871 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1872 MI.getOperand(2).getReg() != AArch64::XZR)
1882 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1886 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1888 if (CmpValue && !IsSubsRegImm)
1890 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1895 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1898 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1902 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1905 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1906 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1909 if (MIUsedNZCV.
N && !CmpValue)
1951bool AArch64InstrInfo::removeCmpToZeroOrOne(
1959 bool IsInvertCC =
false;
1969 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1980 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1981 MI.getOpcode() != AArch64::CATCHRET)
1989 if (
MI.getOpcode() == AArch64::CATCHRET) {
1998 FirstEpilogSEH = std::prev(FirstEpilogSEH);
2000 FirstEpilogSEH = std::next(FirstEpilogSEH);
2015 if (M.getStackProtectorGuard() ==
"sysreg") {
2025 int Offset = M.getStackProtectorGuardOffset();
2077 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2086 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2128 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2151 switch (
MI.getOpcode()) {
2154 case AArch64::MOVZWi:
2155 case AArch64::MOVZXi:
2156 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2157 assert(
MI.getDesc().getNumOperands() == 3 &&
2158 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2162 case AArch64::ANDWri:
2163 return MI.getOperand(1).getReg() == AArch64::WZR;
2164 case AArch64::ANDXri:
2165 return MI.getOperand(1).getReg() == AArch64::XZR;
2166 case TargetOpcode::COPY:
2167 return MI.getOperand(1).getReg() == AArch64::WZR;
2175 switch (
MI.getOpcode()) {
2178 case TargetOpcode::COPY: {
2181 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2182 AArch64::GPR64RegClass.
contains(DstReg));
2184 case AArch64::ORRXrs:
2185 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2186 assert(
MI.getDesc().getNumOperands() == 4 &&
2187 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2191 case AArch64::ADDXri:
2192 if (
MI.getOperand(2).getImm() == 0) {
2193 assert(
MI.getDesc().getNumOperands() == 4 &&
2194 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2205 switch (
MI.getOpcode()) {
2208 case TargetOpcode::COPY: {
2210 return AArch64::FPR128RegClass.contains(DstReg);
2212 case AArch64::ORRv16i8:
2213 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2214 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2215 "invalid ORRv16i8 operands");
2224 int &FrameIndex)
const {
2225 switch (
MI.getOpcode()) {
2228 case AArch64::LDRWui:
2229 case AArch64::LDRXui:
2230 case AArch64::LDRBui:
2231 case AArch64::LDRHui:
2232 case AArch64::LDRSui:
2233 case AArch64::LDRDui:
2234 case AArch64::LDRQui:
2235 case AArch64::LDR_PXI:
2236 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2237 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2238 FrameIndex =
MI.getOperand(1).getIndex();
2239 return MI.getOperand(0).getReg();
2248 int &FrameIndex)
const {
2249 switch (
MI.getOpcode()) {
2252 case AArch64::STRWui:
2253 case AArch64::STRXui:
2254 case AArch64::STRBui:
2255 case AArch64::STRHui:
2256 case AArch64::STRSui:
2257 case AArch64::STRDui:
2258 case AArch64::STRQui:
2259 case AArch64::STR_PXI:
2260 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2261 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2262 FrameIndex =
MI.getOperand(1).getIndex();
2263 return MI.getOperand(0).getReg();
2273 return MMO->getFlags() & MOSuppressPair;
2279 if (
MI.memoperands_empty())
2287 return MMO->getFlags() & MOStridedAccess;
2295 case AArch64::STURSi:
2296 case AArch64::STRSpre:
2297 case AArch64::STURDi:
2298 case AArch64::STRDpre:
2299 case AArch64::STURQi:
2300 case AArch64::STRQpre:
2301 case AArch64::STURBBi:
2302 case AArch64::STURHHi:
2303 case AArch64::STURWi:
2304 case AArch64::STRWpre:
2305 case AArch64::STURXi:
2306 case AArch64::STRXpre:
2307 case AArch64::LDURSi:
2308 case AArch64::LDRSpre:
2309 case AArch64::LDURDi:
2310 case AArch64::LDRDpre:
2311 case AArch64::LDURQi:
2312 case AArch64::LDRQpre:
2313 case AArch64::LDURWi:
2314 case AArch64::LDRWpre:
2315 case AArch64::LDURXi:
2316 case AArch64::LDRXpre:
2317 case AArch64::LDRSWpre:
2318 case AArch64::LDURSWi:
2319 case AArch64::LDURHHi:
2320 case AArch64::LDURBBi:
2321 case AArch64::LDURSBWi:
2322 case AArch64::LDURSHWi:
2330 case AArch64::PRFMui:
return AArch64::PRFUMi;
2331 case AArch64::LDRXui:
return AArch64::LDURXi;
2332 case AArch64::LDRWui:
return AArch64::LDURWi;
2333 case AArch64::LDRBui:
return AArch64::LDURBi;
2334 case AArch64::LDRHui:
return AArch64::LDURHi;
2335 case AArch64::LDRSui:
return AArch64::LDURSi;
2336 case AArch64::LDRDui:
return AArch64::LDURDi;
2337 case AArch64::LDRQui:
return AArch64::LDURQi;
2338 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2339 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2340 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2341 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2342 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2343 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2344 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2345 case AArch64::STRXui:
return AArch64::STURXi;
2346 case AArch64::STRWui:
return AArch64::STURWi;
2347 case AArch64::STRBui:
return AArch64::STURBi;
2348 case AArch64::STRHui:
return AArch64::STURHi;
2349 case AArch64::STRSui:
return AArch64::STURSi;
2350 case AArch64::STRDui:
return AArch64::STURDi;
2351 case AArch64::STRQui:
return AArch64::STURQi;
2352 case AArch64::STRBBui:
return AArch64::STURBBi;
2353 case AArch64::STRHHui:
return AArch64::STURHHi;
2362 case AArch64::LDAPURBi:
2363 case AArch64::LDAPURHi:
2364 case AArch64::LDAPURi:
2365 case AArch64::LDAPURSBWi:
2366 case AArch64::LDAPURSBXi:
2367 case AArch64::LDAPURSHWi:
2368 case AArch64::LDAPURSHXi:
2369 case AArch64::LDAPURSWi:
2370 case AArch64::LDAPURXi:
2371 case AArch64::LDR_PPXI:
2372 case AArch64::LDR_PXI:
2373 case AArch64::LDR_ZXI:
2374 case AArch64::LDR_ZZXI:
2375 case AArch64::LDR_ZZZXI:
2376 case AArch64::LDR_ZZZZXI:
2377 case AArch64::LDRBBui:
2378 case AArch64::LDRBui:
2379 case AArch64::LDRDui:
2380 case AArch64::LDRHHui:
2381 case AArch64::LDRHui:
2382 case AArch64::LDRQui:
2383 case AArch64::LDRSBWui:
2384 case AArch64::LDRSBXui:
2385 case AArch64::LDRSHWui:
2386 case AArch64::LDRSHXui:
2387 case AArch64::LDRSui:
2388 case AArch64::LDRSWui:
2389 case AArch64::LDRWui:
2390 case AArch64::LDRXui:
2391 case AArch64::LDURBBi:
2392 case AArch64::LDURBi:
2393 case AArch64::LDURDi:
2394 case AArch64::LDURHHi:
2395 case AArch64::LDURHi:
2396 case AArch64::LDURQi:
2397 case AArch64::LDURSBWi:
2398 case AArch64::LDURSBXi:
2399 case AArch64::LDURSHWi:
2400 case AArch64::LDURSHXi:
2401 case AArch64::LDURSi:
2402 case AArch64::LDURSWi:
2403 case AArch64::LDURWi:
2404 case AArch64::LDURXi:
2405 case AArch64::PRFMui:
2406 case AArch64::PRFUMi:
2407 case AArch64::ST2Gi:
2409 case AArch64::STLURBi:
2410 case AArch64::STLURHi:
2411 case AArch64::STLURWi:
2412 case AArch64::STLURXi:
2413 case AArch64::StoreSwiftAsyncContext:
2414 case AArch64::STR_PPXI:
2415 case AArch64::STR_PXI:
2416 case AArch64::STR_ZXI:
2417 case AArch64::STR_ZZXI:
2418 case AArch64::STR_ZZZXI:
2419 case AArch64::STR_ZZZZXI:
2420 case AArch64::STRBBui:
2421 case AArch64::STRBui:
2422 case AArch64::STRDui:
2423 case AArch64::STRHHui:
2424 case AArch64::STRHui:
2425 case AArch64::STRQui:
2426 case AArch64::STRSui:
2427 case AArch64::STRWui:
2428 case AArch64::STRXui:
2429 case AArch64::STURBBi:
2430 case AArch64::STURBi:
2431 case AArch64::STURDi:
2432 case AArch64::STURHHi:
2433 case AArch64::STURHi:
2434 case AArch64::STURQi:
2435 case AArch64::STURSi:
2436 case AArch64::STURWi:
2437 case AArch64::STURXi:
2438 case AArch64::STZ2Gi:
2439 case AArch64::STZGi:
2440 case AArch64::TAGPstack:
2442 case AArch64::LD1B_D_IMM:
2443 case AArch64::LD1B_H_IMM:
2444 case AArch64::LD1B_IMM:
2445 case AArch64::LD1B_S_IMM:
2446 case AArch64::LD1D_IMM:
2447 case AArch64::LD1H_D_IMM:
2448 case AArch64::LD1H_IMM:
2449 case AArch64::LD1H_S_IMM:
2450 case AArch64::LD1RB_D_IMM:
2451 case AArch64::LD1RB_H_IMM:
2452 case AArch64::LD1RB_IMM:
2453 case AArch64::LD1RB_S_IMM:
2454 case AArch64::LD1RD_IMM:
2455 case AArch64::LD1RH_D_IMM:
2456 case AArch64::LD1RH_IMM:
2457 case AArch64::LD1RH_S_IMM:
2458 case AArch64::LD1RSB_D_IMM:
2459 case AArch64::LD1RSB_H_IMM:
2460 case AArch64::LD1RSB_S_IMM:
2461 case AArch64::LD1RSH_D_IMM:
2462 case AArch64::LD1RSH_S_IMM:
2463 case AArch64::LD1RSW_IMM:
2464 case AArch64::LD1RW_D_IMM:
2465 case AArch64::LD1RW_IMM:
2466 case AArch64::LD1SB_D_IMM:
2467 case AArch64::LD1SB_H_IMM:
2468 case AArch64::LD1SB_S_IMM:
2469 case AArch64::LD1SH_D_IMM:
2470 case AArch64::LD1SH_S_IMM:
2471 case AArch64::LD1SW_D_IMM:
2472 case AArch64::LD1W_D_IMM:
2473 case AArch64::LD1W_IMM:
2474 case AArch64::LD2B_IMM:
2475 case AArch64::LD2D_IMM:
2476 case AArch64::LD2H_IMM:
2477 case AArch64::LD2W_IMM:
2478 case AArch64::LD3B_IMM:
2479 case AArch64::LD3D_IMM:
2480 case AArch64::LD3H_IMM:
2481 case AArch64::LD3W_IMM:
2482 case AArch64::LD4B_IMM:
2483 case AArch64::LD4D_IMM:
2484 case AArch64::LD4H_IMM:
2485 case AArch64::LD4W_IMM:
2487 case AArch64::LDNF1B_D_IMM:
2488 case AArch64::LDNF1B_H_IMM:
2489 case AArch64::LDNF1B_IMM:
2490 case AArch64::LDNF1B_S_IMM:
2491 case AArch64::LDNF1D_IMM:
2492 case AArch64::LDNF1H_D_IMM:
2493 case AArch64::LDNF1H_IMM:
2494 case AArch64::LDNF1H_S_IMM:
2495 case AArch64::LDNF1SB_D_IMM:
2496 case AArch64::LDNF1SB_H_IMM:
2497 case AArch64::LDNF1SB_S_IMM:
2498 case AArch64::LDNF1SH_D_IMM:
2499 case AArch64::LDNF1SH_S_IMM:
2500 case AArch64::LDNF1SW_D_IMM:
2501 case AArch64::LDNF1W_D_IMM:
2502 case AArch64::LDNF1W_IMM:
2503 case AArch64::LDNPDi:
2504 case AArch64::LDNPQi:
2505 case AArch64::LDNPSi:
2506 case AArch64::LDNPWi:
2507 case AArch64::LDNPXi:
2508 case AArch64::LDNT1B_ZRI:
2509 case AArch64::LDNT1D_ZRI:
2510 case AArch64::LDNT1H_ZRI:
2511 case AArch64::LDNT1W_ZRI:
2512 case AArch64::LDPDi:
2513 case AArch64::LDPQi:
2514 case AArch64::LDPSi:
2515 case AArch64::LDPWi:
2516 case AArch64::LDPXi:
2517 case AArch64::LDRBBpost:
2518 case AArch64::LDRBBpre:
2519 case AArch64::LDRBpost:
2520 case AArch64::LDRBpre:
2521 case AArch64::LDRDpost:
2522 case AArch64::LDRDpre:
2523 case AArch64::LDRHHpost:
2524 case AArch64::LDRHHpre:
2525 case AArch64::LDRHpost:
2526 case AArch64::LDRHpre:
2527 case AArch64::LDRQpost:
2528 case AArch64::LDRQpre:
2529 case AArch64::LDRSpost:
2530 case AArch64::LDRSpre:
2531 case AArch64::LDRWpost:
2532 case AArch64::LDRWpre:
2533 case AArch64::LDRXpost:
2534 case AArch64::LDRXpre:
2535 case AArch64::ST1B_D_IMM:
2536 case AArch64::ST1B_H_IMM:
2537 case AArch64::ST1B_IMM:
2538 case AArch64::ST1B_S_IMM:
2539 case AArch64::ST1D_IMM:
2540 case AArch64::ST1H_D_IMM:
2541 case AArch64::ST1H_IMM:
2542 case AArch64::ST1H_S_IMM:
2543 case AArch64::ST1W_D_IMM:
2544 case AArch64::ST1W_IMM:
2545 case AArch64::ST2B_IMM:
2546 case AArch64::ST2D_IMM:
2547 case AArch64::ST2H_IMM:
2548 case AArch64::ST2W_IMM:
2549 case AArch64::ST3B_IMM:
2550 case AArch64::ST3D_IMM:
2551 case AArch64::ST3H_IMM:
2552 case AArch64::ST3W_IMM:
2553 case AArch64::ST4B_IMM:
2554 case AArch64::ST4D_IMM:
2555 case AArch64::ST4H_IMM:
2556 case AArch64::ST4W_IMM:
2557 case AArch64::STGPi:
2558 case AArch64::STGPreIndex:
2559 case AArch64::STZGPreIndex:
2560 case AArch64::ST2GPreIndex:
2561 case AArch64::STZ2GPreIndex:
2562 case AArch64::STGPostIndex:
2563 case AArch64::STZGPostIndex:
2564 case AArch64::ST2GPostIndex:
2565 case AArch64::STZ2GPostIndex:
2566 case AArch64::STNPDi:
2567 case AArch64::STNPQi:
2568 case AArch64::STNPSi:
2569 case AArch64::STNPWi:
2570 case AArch64::STNPXi:
2571 case AArch64::STNT1B_ZRI:
2572 case AArch64::STNT1D_ZRI:
2573 case AArch64::STNT1H_ZRI:
2574 case AArch64::STNT1W_ZRI:
2575 case AArch64::STPDi:
2576 case AArch64::STPQi:
2577 case AArch64::STPSi:
2578 case AArch64::STPWi:
2579 case AArch64::STPXi:
2580 case AArch64::STRBBpost:
2581 case AArch64::STRBBpre:
2582 case AArch64::STRBpost:
2583 case AArch64::STRBpre:
2584 case AArch64::STRDpost:
2585 case AArch64::STRDpre:
2586 case AArch64::STRHHpost:
2587 case AArch64::STRHHpre:
2588 case AArch64::STRHpost:
2589 case AArch64::STRHpre:
2590 case AArch64::STRQpost:
2591 case AArch64::STRQpre:
2592 case AArch64::STRSpost:
2593 case AArch64::STRSpre:
2594 case AArch64::STRWpost:
2595 case AArch64::STRWpre:
2596 case AArch64::STRXpost:
2597 case AArch64::STRXpre:
2599 case AArch64::LDPDpost:
2600 case AArch64::LDPDpre:
2601 case AArch64::LDPQpost:
2602 case AArch64::LDPQpre:
2603 case AArch64::LDPSpost:
2604 case AArch64::LDPSpre:
2605 case AArch64::LDPWpost:
2606 case AArch64::LDPWpre:
2607 case AArch64::LDPXpost:
2608 case AArch64::LDPXpre:
2609 case AArch64::STGPpre:
2610 case AArch64::STGPpost:
2611 case AArch64::STPDpost:
2612 case AArch64::STPDpre:
2613 case AArch64::STPQpost:
2614 case AArch64::STPQpre:
2615 case AArch64::STPSpost:
2616 case AArch64::STPSpre:
2617 case AArch64::STPWpost:
2618 case AArch64::STPWpre:
2619 case AArch64::STPXpost:
2620 case AArch64::STPXpre:
2626 switch (
MI.getOpcode()) {
2630 case AArch64::STRSui:
2631 case AArch64::STRDui:
2632 case AArch64::STRQui:
2633 case AArch64::STRXui:
2634 case AArch64::STRWui:
2635 case AArch64::LDRSui:
2636 case AArch64::LDRDui:
2637 case AArch64::LDRQui:
2638 case AArch64::LDRXui:
2639 case AArch64::LDRWui:
2640 case AArch64::LDRSWui:
2642 case AArch64::STURSi:
2643 case AArch64::STRSpre:
2644 case AArch64::STURDi:
2645 case AArch64::STRDpre:
2646 case AArch64::STURQi:
2647 case AArch64::STRQpre:
2648 case AArch64::STURWi:
2649 case AArch64::STRWpre:
2650 case AArch64::STURXi:
2651 case AArch64::STRXpre:
2652 case AArch64::LDURSi:
2653 case AArch64::LDRSpre:
2654 case AArch64::LDURDi:
2655 case AArch64::LDRDpre:
2656 case AArch64::LDURQi:
2657 case AArch64::LDRQpre:
2658 case AArch64::LDURWi:
2659 case AArch64::LDRWpre:
2660 case AArch64::LDURXi:
2661 case AArch64::LDRXpre:
2662 case AArch64::LDURSWi:
2663 case AArch64::LDRSWpre:
2669 switch (
MI.getOpcode()) {
2672 "Unexpected instruction - was a new tail call opcode introduced?");
2674 case AArch64::TCRETURNdi:
2675 case AArch64::TCRETURNri:
2676 case AArch64::TCRETURNrix16x17:
2677 case AArch64::TCRETURNrix17:
2678 case AArch64::TCRETURNrinotx16:
2679 case AArch64::TCRETURNriALL:
2680 case AArch64::AUTH_TCRETURN:
2681 case AArch64::AUTH_TCRETURN_BTI:
2691 case AArch64::ADDWri:
2692 return AArch64::ADDSWri;
2693 case AArch64::ADDWrr:
2694 return AArch64::ADDSWrr;
2695 case AArch64::ADDWrs:
2696 return AArch64::ADDSWrs;
2697 case AArch64::ADDWrx:
2698 return AArch64::ADDSWrx;
2699 case AArch64::ANDWri:
2700 return AArch64::ANDSWri;
2701 case AArch64::ANDWrr:
2702 return AArch64::ANDSWrr;
2703 case AArch64::ANDWrs:
2704 return AArch64::ANDSWrs;
2705 case AArch64::BICWrr:
2706 return AArch64::BICSWrr;
2707 case AArch64::BICWrs:
2708 return AArch64::BICSWrs;
2709 case AArch64::SUBWri:
2710 return AArch64::SUBSWri;
2711 case AArch64::SUBWrr:
2712 return AArch64::SUBSWrr;
2713 case AArch64::SUBWrs:
2714 return AArch64::SUBSWrs;
2715 case AArch64::SUBWrx:
2716 return AArch64::SUBSWrx;
2718 case AArch64::ADDXri:
2719 return AArch64::ADDSXri;
2720 case AArch64::ADDXrr:
2721 return AArch64::ADDSXrr;
2722 case AArch64::ADDXrs:
2723 return AArch64::ADDSXrs;
2724 case AArch64::ADDXrx:
2725 return AArch64::ADDSXrx;
2726 case AArch64::ANDXri:
2727 return AArch64::ANDSXri;
2728 case AArch64::ANDXrr:
2729 return AArch64::ANDSXrr;
2730 case AArch64::ANDXrs:
2731 return AArch64::ANDSXrs;
2732 case AArch64::BICXrr:
2733 return AArch64::BICSXrr;
2734 case AArch64::BICXrs:
2735 return AArch64::BICSXrs;
2736 case AArch64::SUBXri:
2737 return AArch64::SUBSXri;
2738 case AArch64::SUBXrr:
2739 return AArch64::SUBSXrr;
2740 case AArch64::SUBXrs:
2741 return AArch64::SUBSXrs;
2742 case AArch64::SUBXrx:
2743 return AArch64::SUBSXrx;
2745 case AArch64::AND_PPzPP:
2746 return AArch64::ANDS_PPzPP;
2747 case AArch64::BIC_PPzPP:
2748 return AArch64::BICS_PPzPP;
2749 case AArch64::EOR_PPzPP:
2750 return AArch64::EORS_PPzPP;
2751 case AArch64::NAND_PPzPP:
2752 return AArch64::NANDS_PPzPP;
2753 case AArch64::NOR_PPzPP:
2754 return AArch64::NORS_PPzPP;
2755 case AArch64::ORN_PPzPP:
2756 return AArch64::ORNS_PPzPP;
2757 case AArch64::ORR_PPzPP:
2758 return AArch64::ORRS_PPzPP;
2759 case AArch64::BRKA_PPzP:
2760 return AArch64::BRKAS_PPzP;
2761 case AArch64::BRKPA_PPzPP:
2762 return AArch64::BRKPAS_PPzPP;
2763 case AArch64::BRKB_PPzP:
2764 return AArch64::BRKBS_PPzP;
2765 case AArch64::BRKPB_PPzPP:
2766 return AArch64::BRKPBS_PPzPP;
2767 case AArch64::BRKN_PPzP:
2768 return AArch64::BRKNS_PPzP;
2769 case AArch64::RDFFR_PPz:
2770 return AArch64::RDFFRS_PPz;
2771 case AArch64::PTRUE_B:
2772 return AArch64::PTRUES_B;
2783 if (
MI.hasOrderedMemoryRef())
2788 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2789 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2790 "Expected a reg or frame index operand.");
2794 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2796 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2809 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2810 Register BaseReg =
MI.getOperand(1).getReg();
2812 if (
MI.modifiesRegister(BaseReg,
TRI))
2825 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2827 MI.getMF()->getFunction().needsUnwindTableEntry();
2833 if (Subtarget.isPaired128Slow()) {
2834 switch (
MI.getOpcode()) {
2837 case AArch64::LDURQi:
2838 case AArch64::STURQi:
2839 case AArch64::LDRQui:
2840 case AArch64::STRQui:
2867std::optional<ExtAddrMode>
2872 bool OffsetIsScalable;
2873 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2874 return std::nullopt;
2877 return std::nullopt;
2892 int64_t OffsetScale = 1;
2897 case AArch64::LDURQi:
2898 case AArch64::STURQi:
2902 case AArch64::LDURDi:
2903 case AArch64::STURDi:
2904 case AArch64::LDURXi:
2905 case AArch64::STURXi:
2909 case AArch64::LDURWi:
2910 case AArch64::LDURSWi:
2911 case AArch64::STURWi:
2915 case AArch64::LDURHi:
2916 case AArch64::STURHi:
2917 case AArch64::LDURHHi:
2918 case AArch64::STURHHi:
2919 case AArch64::LDURSHXi:
2920 case AArch64::LDURSHWi:
2924 case AArch64::LDRBroX:
2925 case AArch64::LDRBBroX:
2926 case AArch64::LDRSBXroX:
2927 case AArch64::LDRSBWroX:
2928 case AArch64::STRBroX:
2929 case AArch64::STRBBroX:
2930 case AArch64::LDURBi:
2931 case AArch64::LDURBBi:
2932 case AArch64::LDURSBXi:
2933 case AArch64::LDURSBWi:
2934 case AArch64::STURBi:
2935 case AArch64::STURBBi:
2936 case AArch64::LDRBui:
2937 case AArch64::LDRBBui:
2938 case AArch64::LDRSBXui:
2939 case AArch64::LDRSBWui:
2940 case AArch64::STRBui:
2941 case AArch64::STRBBui:
2945 case AArch64::LDRQroX:
2946 case AArch64::STRQroX:
2947 case AArch64::LDRQui:
2948 case AArch64::STRQui:
2953 case AArch64::LDRDroX:
2954 case AArch64::STRDroX:
2955 case AArch64::LDRXroX:
2956 case AArch64::STRXroX:
2957 case AArch64::LDRDui:
2958 case AArch64::STRDui:
2959 case AArch64::LDRXui:
2960 case AArch64::STRXui:
2965 case AArch64::LDRWroX:
2966 case AArch64::LDRSWroX:
2967 case AArch64::STRWroX:
2968 case AArch64::LDRWui:
2969 case AArch64::LDRSWui:
2970 case AArch64::STRWui:
2975 case AArch64::LDRHroX:
2976 case AArch64::STRHroX:
2977 case AArch64::LDRHHroX:
2978 case AArch64::STRHHroX:
2979 case AArch64::LDRSHXroX:
2980 case AArch64::LDRSHWroX:
2981 case AArch64::LDRHui:
2982 case AArch64::STRHui:
2983 case AArch64::LDRHHui:
2984 case AArch64::STRHHui:
2985 case AArch64::LDRSHXui:
2986 case AArch64::LDRSHWui:
2994 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
3018 case AArch64::SBFMXri:
3031 AM.
Scale = OffsetScale;
3036 case TargetOpcode::SUBREG_TO_REG: {
3049 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3053 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3055 DefMI.getOperand(3).getImm() != 0)
3062 AM.
Scale = OffsetScale;
3073 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3074 int64_t NewOffset) ->
bool {
3075 int64_t MinOffset, MaxOffset;
3092 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3093 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3095 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3097 int64_t NewOffset = OldOffset + Disp;
3102 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3112 auto canFoldAddRegIntoAddrMode =
3129 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3130 Subtarget.isSTRQroSlow();
3139 case AArch64::ADDXri:
3145 return canFoldAddSubImmIntoAddrMode(Disp);
3147 case AArch64::SUBXri:
3153 return canFoldAddSubImmIntoAddrMode(-Disp);
3155 case AArch64::ADDXrs: {
3168 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3170 if (avoidSlowSTRQ(MemI))
3173 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3176 case AArch64::ADDXrr:
3184 if (!OptSize && avoidSlowSTRQ(MemI))
3186 return canFoldAddRegIntoAddrMode(1);
3188 case AArch64::ADDXrx:
3196 if (!OptSize && avoidSlowSTRQ(MemI))
3205 return canFoldAddRegIntoAddrMode(
3220 case AArch64::LDURQi:
3221 case AArch64::LDRQui:
3222 return AArch64::LDRQroX;
3223 case AArch64::STURQi:
3224 case AArch64::STRQui:
3225 return AArch64::STRQroX;
3226 case AArch64::LDURDi:
3227 case AArch64::LDRDui:
3228 return AArch64::LDRDroX;
3229 case AArch64::STURDi:
3230 case AArch64::STRDui:
3231 return AArch64::STRDroX;
3232 case AArch64::LDURXi:
3233 case AArch64::LDRXui:
3234 return AArch64::LDRXroX;
3235 case AArch64::STURXi:
3236 case AArch64::STRXui:
3237 return AArch64::STRXroX;
3238 case AArch64::LDURWi:
3239 case AArch64::LDRWui:
3240 return AArch64::LDRWroX;
3241 case AArch64::LDURSWi:
3242 case AArch64::LDRSWui:
3243 return AArch64::LDRSWroX;
3244 case AArch64::STURWi:
3245 case AArch64::STRWui:
3246 return AArch64::STRWroX;
3247 case AArch64::LDURHi:
3248 case AArch64::LDRHui:
3249 return AArch64::LDRHroX;
3250 case AArch64::STURHi:
3251 case AArch64::STRHui:
3252 return AArch64::STRHroX;
3253 case AArch64::LDURHHi:
3254 case AArch64::LDRHHui:
3255 return AArch64::LDRHHroX;
3256 case AArch64::STURHHi:
3257 case AArch64::STRHHui:
3258 return AArch64::STRHHroX;
3259 case AArch64::LDURSHXi:
3260 case AArch64::LDRSHXui:
3261 return AArch64::LDRSHXroX;
3262 case AArch64::LDURSHWi:
3263 case AArch64::LDRSHWui:
3264 return AArch64::LDRSHWroX;
3265 case AArch64::LDURBi:
3266 case AArch64::LDRBui:
3267 return AArch64::LDRBroX;
3268 case AArch64::LDURBBi:
3269 case AArch64::LDRBBui:
3270 return AArch64::LDRBBroX;
3271 case AArch64::LDURSBXi:
3272 case AArch64::LDRSBXui:
3273 return AArch64::LDRSBXroX;
3274 case AArch64::LDURSBWi:
3275 case AArch64::LDRSBWui:
3276 return AArch64::LDRSBWroX;
3277 case AArch64::STURBi:
3278 case AArch64::STRBui:
3279 return AArch64::STRBroX;
3280 case AArch64::STURBBi:
3281 case AArch64::STRBBui:
3282 return AArch64::STRBBroX;
3294 case AArch64::LDURQi:
3296 return AArch64::LDRQui;
3297 case AArch64::STURQi:
3299 return AArch64::STRQui;
3300 case AArch64::LDURDi:
3302 return AArch64::LDRDui;
3303 case AArch64::STURDi:
3305 return AArch64::STRDui;
3306 case AArch64::LDURXi:
3308 return AArch64::LDRXui;
3309 case AArch64::STURXi:
3311 return AArch64::STRXui;
3312 case AArch64::LDURWi:
3314 return AArch64::LDRWui;
3315 case AArch64::LDURSWi:
3317 return AArch64::LDRSWui;
3318 case AArch64::STURWi:
3320 return AArch64::STRWui;
3321 case AArch64::LDURHi:
3323 return AArch64::LDRHui;
3324 case AArch64::STURHi:
3326 return AArch64::STRHui;
3327 case AArch64::LDURHHi:
3329 return AArch64::LDRHHui;
3330 case AArch64::STURHHi:
3332 return AArch64::STRHHui;
3333 case AArch64::LDURSHXi:
3335 return AArch64::LDRSHXui;
3336 case AArch64::LDURSHWi:
3338 return AArch64::LDRSHWui;
3339 case AArch64::LDURBi:
3341 return AArch64::LDRBui;
3342 case AArch64::LDURBBi:
3344 return AArch64::LDRBBui;
3345 case AArch64::LDURSBXi:
3347 return AArch64::LDRSBXui;
3348 case AArch64::LDURSBWi:
3350 return AArch64::LDRSBWui;
3351 case AArch64::STURBi:
3353 return AArch64::STRBui;
3354 case AArch64::STURBBi:
3356 return AArch64::STRBBui;
3357 case AArch64::LDRQui:
3358 case AArch64::STRQui:
3361 case AArch64::LDRDui:
3362 case AArch64::STRDui:
3363 case AArch64::LDRXui:
3364 case AArch64::STRXui:
3367 case AArch64::LDRWui:
3368 case AArch64::LDRSWui:
3369 case AArch64::STRWui:
3372 case AArch64::LDRHui:
3373 case AArch64::STRHui:
3374 case AArch64::LDRHHui:
3375 case AArch64::STRHHui:
3376 case AArch64::LDRSHXui:
3377 case AArch64::LDRSHWui:
3380 case AArch64::LDRBui:
3381 case AArch64::LDRBBui:
3382 case AArch64::LDRSBXui:
3383 case AArch64::LDRSBWui:
3384 case AArch64::STRBui:
3385 case AArch64::STRBBui:
3399 case AArch64::LDURQi:
3400 case AArch64::STURQi:
3401 case AArch64::LDURDi:
3402 case AArch64::STURDi:
3403 case AArch64::LDURXi:
3404 case AArch64::STURXi:
3405 case AArch64::LDURWi:
3406 case AArch64::LDURSWi:
3407 case AArch64::STURWi:
3408 case AArch64::LDURHi:
3409 case AArch64::STURHi:
3410 case AArch64::LDURHHi:
3411 case AArch64::STURHHi:
3412 case AArch64::LDURSHXi:
3413 case AArch64::LDURSHWi:
3414 case AArch64::LDURBi:
3415 case AArch64::STURBi:
3416 case AArch64::LDURBBi:
3417 case AArch64::STURBBi:
3418 case AArch64::LDURSBWi:
3419 case AArch64::LDURSBXi:
3421 case AArch64::LDRQui:
3422 return AArch64::LDURQi;
3423 case AArch64::STRQui:
3424 return AArch64::STURQi;
3425 case AArch64::LDRDui:
3426 return AArch64::LDURDi;
3427 case AArch64::STRDui:
3428 return AArch64::STURDi;
3429 case AArch64::LDRXui:
3430 return AArch64::LDURXi;
3431 case AArch64::STRXui:
3432 return AArch64::STURXi;
3433 case AArch64::LDRWui:
3434 return AArch64::LDURWi;
3435 case AArch64::LDRSWui:
3436 return AArch64::LDURSWi;
3437 case AArch64::STRWui:
3438 return AArch64::STURWi;
3439 case AArch64::LDRHui:
3440 return AArch64::LDURHi;
3441 case AArch64::STRHui:
3442 return AArch64::STURHi;
3443 case AArch64::LDRHHui:
3444 return AArch64::LDURHHi;
3445 case AArch64::STRHHui:
3446 return AArch64::STURHHi;
3447 case AArch64::LDRSHXui:
3448 return AArch64::LDURSHXi;
3449 case AArch64::LDRSHWui:
3450 return AArch64::LDURSHWi;
3451 case AArch64::LDRBBui:
3452 return AArch64::LDURBBi;
3453 case AArch64::LDRBui:
3454 return AArch64::LDURBi;
3455 case AArch64::STRBBui:
3456 return AArch64::STURBBi;
3457 case AArch64::STRBui:
3458 return AArch64::STURBi;
3459 case AArch64::LDRSBWui:
3460 return AArch64::LDURSBWi;
3461 case AArch64::LDRSBXui:
3462 return AArch64::LDURSBXi;
3475 case AArch64::LDRQroX:
3476 case AArch64::LDURQi:
3477 case AArch64::LDRQui:
3478 return AArch64::LDRQroW;
3479 case AArch64::STRQroX:
3480 case AArch64::STURQi:
3481 case AArch64::STRQui:
3482 return AArch64::STRQroW;
3483 case AArch64::LDRDroX:
3484 case AArch64::LDURDi:
3485 case AArch64::LDRDui:
3486 return AArch64::LDRDroW;
3487 case AArch64::STRDroX:
3488 case AArch64::STURDi:
3489 case AArch64::STRDui:
3490 return AArch64::STRDroW;
3491 case AArch64::LDRXroX:
3492 case AArch64::LDURXi:
3493 case AArch64::LDRXui:
3494 return AArch64::LDRXroW;
3495 case AArch64::STRXroX:
3496 case AArch64::STURXi:
3497 case AArch64::STRXui:
3498 return AArch64::STRXroW;
3499 case AArch64::LDRWroX:
3500 case AArch64::LDURWi:
3501 case AArch64::LDRWui:
3502 return AArch64::LDRWroW;
3503 case AArch64::LDRSWroX:
3504 case AArch64::LDURSWi:
3505 case AArch64::LDRSWui:
3506 return AArch64::LDRSWroW;
3507 case AArch64::STRWroX:
3508 case AArch64::STURWi:
3509 case AArch64::STRWui:
3510 return AArch64::STRWroW;
3511 case AArch64::LDRHroX:
3512 case AArch64::LDURHi:
3513 case AArch64::LDRHui:
3514 return AArch64::LDRHroW;
3515 case AArch64::STRHroX:
3516 case AArch64::STURHi:
3517 case AArch64::STRHui:
3518 return AArch64::STRHroW;
3519 case AArch64::LDRHHroX:
3520 case AArch64::LDURHHi:
3521 case AArch64::LDRHHui:
3522 return AArch64::LDRHHroW;
3523 case AArch64::STRHHroX:
3524 case AArch64::STURHHi:
3525 case AArch64::STRHHui:
3526 return AArch64::STRHHroW;
3527 case AArch64::LDRSHXroX:
3528 case AArch64::LDURSHXi:
3529 case AArch64::LDRSHXui:
3530 return AArch64::LDRSHXroW;
3531 case AArch64::LDRSHWroX:
3532 case AArch64::LDURSHWi:
3533 case AArch64::LDRSHWui:
3534 return AArch64::LDRSHWroW;
3535 case AArch64::LDRBroX:
3536 case AArch64::LDURBi:
3537 case AArch64::LDRBui:
3538 return AArch64::LDRBroW;
3539 case AArch64::LDRBBroX:
3540 case AArch64::LDURBBi:
3541 case AArch64::LDRBBui:
3542 return AArch64::LDRBBroW;
3543 case AArch64::LDRSBXroX:
3544 case AArch64::LDURSBXi:
3545 case AArch64::LDRSBXui:
3546 return AArch64::LDRSBXroW;
3547 case AArch64::LDRSBWroX:
3548 case AArch64::LDURSBWi:
3549 case AArch64::LDRSBWui:
3550 return AArch64::LDRSBWroW;
3551 case AArch64::STRBroX:
3552 case AArch64::STURBi:
3553 case AArch64::STRBui:
3554 return AArch64::STRBroW;
3555 case AArch64::STRBBroX:
3556 case AArch64::STURBBi:
3557 case AArch64::STRBBui:
3558 return AArch64::STRBBroW;
3573 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3583 return B.getInstr();
3587 "Addressing mode not supported for folding");
3604 return B.getInstr();
3611 "Address offset can be a register or an immediate, but not both");
3613 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3618 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3632 return B.getInstr();
3636 "Function must not be called with an addressing mode it can't handle");
3645 case AArch64::LD1Fourv16b_POST:
3646 case AArch64::LD1Fourv1d_POST:
3647 case AArch64::LD1Fourv2d_POST:
3648 case AArch64::LD1Fourv2s_POST:
3649 case AArch64::LD1Fourv4h_POST:
3650 case AArch64::LD1Fourv4s_POST:
3651 case AArch64::LD1Fourv8b_POST:
3652 case AArch64::LD1Fourv8h_POST:
3653 case AArch64::LD1Onev16b_POST:
3654 case AArch64::LD1Onev1d_POST:
3655 case AArch64::LD1Onev2d_POST:
3656 case AArch64::LD1Onev2s_POST:
3657 case AArch64::LD1Onev4h_POST:
3658 case AArch64::LD1Onev4s_POST:
3659 case AArch64::LD1Onev8b_POST:
3660 case AArch64::LD1Onev8h_POST:
3661 case AArch64::LD1Rv16b_POST:
3662 case AArch64::LD1Rv1d_POST:
3663 case AArch64::LD1Rv2d_POST:
3664 case AArch64::LD1Rv2s_POST:
3665 case AArch64::LD1Rv4h_POST:
3666 case AArch64::LD1Rv4s_POST:
3667 case AArch64::LD1Rv8b_POST:
3668 case AArch64::LD1Rv8h_POST:
3669 case AArch64::LD1Threev16b_POST:
3670 case AArch64::LD1Threev1d_POST:
3671 case AArch64::LD1Threev2d_POST:
3672 case AArch64::LD1Threev2s_POST:
3673 case AArch64::LD1Threev4h_POST:
3674 case AArch64::LD1Threev4s_POST:
3675 case AArch64::LD1Threev8b_POST:
3676 case AArch64::LD1Threev8h_POST:
3677 case AArch64::LD1Twov16b_POST:
3678 case AArch64::LD1Twov1d_POST:
3679 case AArch64::LD1Twov2d_POST:
3680 case AArch64::LD1Twov2s_POST:
3681 case AArch64::LD1Twov4h_POST:
3682 case AArch64::LD1Twov4s_POST:
3683 case AArch64::LD1Twov8b_POST:
3684 case AArch64::LD1Twov8h_POST:
3685 case AArch64::LD1i16_POST:
3686 case AArch64::LD1i32_POST:
3687 case AArch64::LD1i64_POST:
3688 case AArch64::LD1i8_POST:
3689 case AArch64::LD2Rv16b_POST:
3690 case AArch64::LD2Rv1d_POST:
3691 case AArch64::LD2Rv2d_POST:
3692 case AArch64::LD2Rv2s_POST:
3693 case AArch64::LD2Rv4h_POST:
3694 case AArch64::LD2Rv4s_POST:
3695 case AArch64::LD2Rv8b_POST:
3696 case AArch64::LD2Rv8h_POST:
3697 case AArch64::LD2Twov16b_POST:
3698 case AArch64::LD2Twov2d_POST:
3699 case AArch64::LD2Twov2s_POST:
3700 case AArch64::LD2Twov4h_POST:
3701 case AArch64::LD2Twov4s_POST:
3702 case AArch64::LD2Twov8b_POST:
3703 case AArch64::LD2Twov8h_POST:
3704 case AArch64::LD2i16_POST:
3705 case AArch64::LD2i32_POST:
3706 case AArch64::LD2i64_POST:
3707 case AArch64::LD2i8_POST:
3708 case AArch64::LD3Rv16b_POST:
3709 case AArch64::LD3Rv1d_POST:
3710 case AArch64::LD3Rv2d_POST:
3711 case AArch64::LD3Rv2s_POST:
3712 case AArch64::LD3Rv4h_POST:
3713 case AArch64::LD3Rv4s_POST:
3714 case AArch64::LD3Rv8b_POST:
3715 case AArch64::LD3Rv8h_POST:
3716 case AArch64::LD3Threev16b_POST:
3717 case AArch64::LD3Threev2d_POST:
3718 case AArch64::LD3Threev2s_POST:
3719 case AArch64::LD3Threev4h_POST:
3720 case AArch64::LD3Threev4s_POST:
3721 case AArch64::LD3Threev8b_POST:
3722 case AArch64::LD3Threev8h_POST:
3723 case AArch64::LD3i16_POST:
3724 case AArch64::LD3i32_POST:
3725 case AArch64::LD3i64_POST:
3726 case AArch64::LD3i8_POST:
3727 case AArch64::LD4Fourv16b_POST:
3728 case AArch64::LD4Fourv2d_POST:
3729 case AArch64::LD4Fourv2s_POST:
3730 case AArch64::LD4Fourv4h_POST:
3731 case AArch64::LD4Fourv4s_POST:
3732 case AArch64::LD4Fourv8b_POST:
3733 case AArch64::LD4Fourv8h_POST:
3734 case AArch64::LD4Rv16b_POST:
3735 case AArch64::LD4Rv1d_POST:
3736 case AArch64::LD4Rv2d_POST:
3737 case AArch64::LD4Rv2s_POST:
3738 case AArch64::LD4Rv4h_POST:
3739 case AArch64::LD4Rv4s_POST:
3740 case AArch64::LD4Rv8b_POST:
3741 case AArch64::LD4Rv8h_POST:
3742 case AArch64::LD4i16_POST:
3743 case AArch64::LD4i32_POST:
3744 case AArch64::LD4i64_POST:
3745 case AArch64::LD4i8_POST:
3746 case AArch64::LDAPRWpost:
3747 case AArch64::LDAPRXpost:
3748 case AArch64::LDIAPPWpost:
3749 case AArch64::LDIAPPXpost:
3750 case AArch64::LDPDpost:
3751 case AArch64::LDPQpost:
3752 case AArch64::LDPSWpost:
3753 case AArch64::LDPSpost:
3754 case AArch64::LDPWpost:
3755 case AArch64::LDPXpost:
3756 case AArch64::LDRBBpost:
3757 case AArch64::LDRBpost:
3758 case AArch64::LDRDpost:
3759 case AArch64::LDRHHpost:
3760 case AArch64::LDRHpost:
3761 case AArch64::LDRQpost:
3762 case AArch64::LDRSBWpost:
3763 case AArch64::LDRSBXpost:
3764 case AArch64::LDRSHWpost:
3765 case AArch64::LDRSHXpost:
3766 case AArch64::LDRSWpost:
3767 case AArch64::LDRSpost:
3768 case AArch64::LDRWpost:
3769 case AArch64::LDRXpost:
3770 case AArch64::ST1Fourv16b_POST:
3771 case AArch64::ST1Fourv1d_POST:
3772 case AArch64::ST1Fourv2d_POST:
3773 case AArch64::ST1Fourv2s_POST:
3774 case AArch64::ST1Fourv4h_POST:
3775 case AArch64::ST1Fourv4s_POST:
3776 case AArch64::ST1Fourv8b_POST:
3777 case AArch64::ST1Fourv8h_POST:
3778 case AArch64::ST1Onev16b_POST:
3779 case AArch64::ST1Onev1d_POST:
3780 case AArch64::ST1Onev2d_POST:
3781 case AArch64::ST1Onev2s_POST:
3782 case AArch64::ST1Onev4h_POST:
3783 case AArch64::ST1Onev4s_POST:
3784 case AArch64::ST1Onev8b_POST:
3785 case AArch64::ST1Onev8h_POST:
3786 case AArch64::ST1Threev16b_POST:
3787 case AArch64::ST1Threev1d_POST:
3788 case AArch64::ST1Threev2d_POST:
3789 case AArch64::ST1Threev2s_POST:
3790 case AArch64::ST1Threev4h_POST:
3791 case AArch64::ST1Threev4s_POST:
3792 case AArch64::ST1Threev8b_POST:
3793 case AArch64::ST1Threev8h_POST:
3794 case AArch64::ST1Twov16b_POST:
3795 case AArch64::ST1Twov1d_POST:
3796 case AArch64::ST1Twov2d_POST:
3797 case AArch64::ST1Twov2s_POST:
3798 case AArch64::ST1Twov4h_POST:
3799 case AArch64::ST1Twov4s_POST:
3800 case AArch64::ST1Twov8b_POST:
3801 case AArch64::ST1Twov8h_POST:
3802 case AArch64::ST1i16_POST:
3803 case AArch64::ST1i32_POST:
3804 case AArch64::ST1i64_POST:
3805 case AArch64::ST1i8_POST:
3806 case AArch64::ST2GPostIndex:
3807 case AArch64::ST2Twov16b_POST:
3808 case AArch64::ST2Twov2d_POST:
3809 case AArch64::ST2Twov2s_POST:
3810 case AArch64::ST2Twov4h_POST:
3811 case AArch64::ST2Twov4s_POST:
3812 case AArch64::ST2Twov8b_POST:
3813 case AArch64::ST2Twov8h_POST:
3814 case AArch64::ST2i16_POST:
3815 case AArch64::ST2i32_POST:
3816 case AArch64::ST2i64_POST:
3817 case AArch64::ST2i8_POST:
3818 case AArch64::ST3Threev16b_POST:
3819 case AArch64::ST3Threev2d_POST:
3820 case AArch64::ST3Threev2s_POST:
3821 case AArch64::ST3Threev4h_POST:
3822 case AArch64::ST3Threev4s_POST:
3823 case AArch64::ST3Threev8b_POST:
3824 case AArch64::ST3Threev8h_POST:
3825 case AArch64::ST3i16_POST:
3826 case AArch64::ST3i32_POST:
3827 case AArch64::ST3i64_POST:
3828 case AArch64::ST3i8_POST:
3829 case AArch64::ST4Fourv16b_POST:
3830 case AArch64::ST4Fourv2d_POST:
3831 case AArch64::ST4Fourv2s_POST:
3832 case AArch64::ST4Fourv4h_POST:
3833 case AArch64::ST4Fourv4s_POST:
3834 case AArch64::ST4Fourv8b_POST:
3835 case AArch64::ST4Fourv8h_POST:
3836 case AArch64::ST4i16_POST:
3837 case AArch64::ST4i32_POST:
3838 case AArch64::ST4i64_POST:
3839 case AArch64::ST4i8_POST:
3840 case AArch64::STGPostIndex:
3841 case AArch64::STGPpost:
3842 case AArch64::STPDpost:
3843 case AArch64::STPQpost:
3844 case AArch64::STPSpost:
3845 case AArch64::STPWpost:
3846 case AArch64::STPXpost:
3847 case AArch64::STRBBpost:
3848 case AArch64::STRBpost:
3849 case AArch64::STRDpost:
3850 case AArch64::STRHHpost:
3851 case AArch64::STRHpost:
3852 case AArch64::STRQpost:
3853 case AArch64::STRSpost:
3854 case AArch64::STRWpost:
3855 case AArch64::STRXpost:
3856 case AArch64::STZ2GPostIndex:
3857 case AArch64::STZGPostIndex:
3864 bool &OffsetIsScalable,
TypeSize &Width,
3885 int64_t Dummy1, Dummy2;
3907 return BaseOp->
isReg() || BaseOp->
isFI();
3914 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3919 TypeSize &Width, int64_t &MinOffset,
3920 int64_t &MaxOffset) {
3926 MinOffset = MaxOffset = 0;
3929 case AArch64::LDRQui:
3930 case AArch64::STRQui:
3936 case AArch64::LDRXui:
3937 case AArch64::LDRDui:
3938 case AArch64::STRXui:
3939 case AArch64::STRDui:
3940 case AArch64::PRFMui:
3946 case AArch64::LDRWui:
3947 case AArch64::LDRSui:
3948 case AArch64::LDRSWui:
3949 case AArch64::STRWui:
3950 case AArch64::STRSui:
3956 case AArch64::LDRHui:
3957 case AArch64::LDRHHui:
3958 case AArch64::LDRSHWui:
3959 case AArch64::LDRSHXui:
3960 case AArch64::STRHui:
3961 case AArch64::STRHHui:
3967 case AArch64::LDRBui:
3968 case AArch64::LDRBBui:
3969 case AArch64::LDRSBWui:
3970 case AArch64::LDRSBXui:
3971 case AArch64::STRBui:
3972 case AArch64::STRBBui:
3979 case AArch64::STRQpre:
3980 case AArch64::LDRQpost:
3986 case AArch64::LDRDpost:
3987 case AArch64::LDRDpre:
3988 case AArch64::LDRXpost:
3989 case AArch64::LDRXpre:
3990 case AArch64::STRDpost:
3991 case AArch64::STRDpre:
3992 case AArch64::STRXpost:
3993 case AArch64::STRXpre:
3999 case AArch64::STRWpost:
4000 case AArch64::STRWpre:
4001 case AArch64::LDRWpost:
4002 case AArch64::LDRWpre:
4003 case AArch64::STRSpost:
4004 case AArch64::STRSpre:
4005 case AArch64::LDRSpost:
4006 case AArch64::LDRSpre:
4012 case AArch64::LDRHpost:
4013 case AArch64::LDRHpre:
4014 case AArch64::STRHpost:
4015 case AArch64::STRHpre:
4016 case AArch64::LDRHHpost:
4017 case AArch64::LDRHHpre:
4018 case AArch64::STRHHpost:
4019 case AArch64::STRHHpre:
4025 case AArch64::LDRBpost:
4026 case AArch64::LDRBpre:
4027 case AArch64::STRBpost:
4028 case AArch64::STRBpre:
4029 case AArch64::LDRBBpost:
4030 case AArch64::LDRBBpre:
4031 case AArch64::STRBBpost:
4032 case AArch64::STRBBpre:
4039 case AArch64::LDURQi:
4040 case AArch64::STURQi:
4046 case AArch64::LDURXi:
4047 case AArch64::LDURDi:
4048 case AArch64::LDAPURXi:
4049 case AArch64::STURXi:
4050 case AArch64::STURDi:
4051 case AArch64::STLURXi:
4052 case AArch64::PRFUMi:
4058 case AArch64::LDURWi:
4059 case AArch64::LDURSi:
4060 case AArch64::LDURSWi:
4061 case AArch64::LDAPURi:
4062 case AArch64::LDAPURSWi:
4063 case AArch64::STURWi:
4064 case AArch64::STURSi:
4065 case AArch64::STLURWi:
4071 case AArch64::LDURHi:
4072 case AArch64::LDURHHi:
4073 case AArch64::LDURSHXi:
4074 case AArch64::LDURSHWi:
4075 case AArch64::LDAPURHi:
4076 case AArch64::LDAPURSHWi:
4077 case AArch64::LDAPURSHXi:
4078 case AArch64::STURHi:
4079 case AArch64::STURHHi:
4080 case AArch64::STLURHi:
4086 case AArch64::LDURBi:
4087 case AArch64::LDURBBi:
4088 case AArch64::LDURSBXi:
4089 case AArch64::LDURSBWi:
4090 case AArch64::LDAPURBi:
4091 case AArch64::LDAPURSBWi:
4092 case AArch64::LDAPURSBXi:
4093 case AArch64::STURBi:
4094 case AArch64::STURBBi:
4095 case AArch64::STLURBi:
4102 case AArch64::LDPQi:
4103 case AArch64::LDNPQi:
4104 case AArch64::STPQi:
4105 case AArch64::STNPQi:
4106 case AArch64::LDPQpost:
4107 case AArch64::LDPQpre:
4108 case AArch64::STPQpost:
4109 case AArch64::STPQpre:
4115 case AArch64::LDPXi:
4116 case AArch64::LDPDi:
4117 case AArch64::LDNPXi:
4118 case AArch64::LDNPDi:
4119 case AArch64::STPXi:
4120 case AArch64::STPDi:
4121 case AArch64::STNPXi:
4122 case AArch64::STNPDi:
4123 case AArch64::LDPDpost:
4124 case AArch64::LDPDpre:
4125 case AArch64::LDPXpost:
4126 case AArch64::LDPXpre:
4127 case AArch64::STPDpost:
4128 case AArch64::STPDpre:
4129 case AArch64::STPXpost:
4130 case AArch64::STPXpre:
4136 case AArch64::LDPWi:
4137 case AArch64::LDPSi:
4138 case AArch64::LDNPWi:
4139 case AArch64::LDNPSi:
4140 case AArch64::STPWi:
4141 case AArch64::STPSi:
4142 case AArch64::STNPWi:
4143 case AArch64::STNPSi:
4144 case AArch64::LDPSpost:
4145 case AArch64::LDPSpre:
4146 case AArch64::LDPWpost:
4147 case AArch64::LDPWpre:
4148 case AArch64::STPSpost:
4149 case AArch64::STPSpre:
4150 case AArch64::STPWpost:
4151 case AArch64::STPWpre:
4157 case AArch64::StoreSwiftAsyncContext:
4170 case AArch64::TAGPstack:
4180 case AArch64::STGPreIndex:
4181 case AArch64::STGPostIndex:
4182 case AArch64::STZGi:
4183 case AArch64::STZGPreIndex:
4184 case AArch64::STZGPostIndex:
4191 case AArch64::STR_ZZZZXI:
4192 case AArch64::LDR_ZZZZXI:
4198 case AArch64::STR_ZZZXI:
4199 case AArch64::LDR_ZZZXI:
4205 case AArch64::STR_ZZXI:
4206 case AArch64::LDR_ZZXI:
4212 case AArch64::LDR_PXI:
4213 case AArch64::STR_PXI:
4219 case AArch64::LDR_PPXI:
4220 case AArch64::STR_PPXI:
4226 case AArch64::LDR_ZXI:
4227 case AArch64::STR_ZXI:
4233 case AArch64::LD1B_IMM:
4234 case AArch64::LD1H_IMM:
4235 case AArch64::LD1W_IMM:
4236 case AArch64::LD1D_IMM:
4237 case AArch64::LDNT1B_ZRI:
4238 case AArch64::LDNT1H_ZRI:
4239 case AArch64::LDNT1W_ZRI:
4240 case AArch64::LDNT1D_ZRI:
4241 case AArch64::ST1B_IMM:
4242 case AArch64::ST1H_IMM:
4243 case AArch64::ST1W_IMM:
4244 case AArch64::ST1D_IMM:
4245 case AArch64::STNT1B_ZRI:
4246 case AArch64::STNT1H_ZRI:
4247 case AArch64::STNT1W_ZRI:
4248 case AArch64::STNT1D_ZRI:
4249 case AArch64::LDNF1B_IMM:
4250 case AArch64::LDNF1H_IMM:
4251 case AArch64::LDNF1W_IMM:
4252 case AArch64::LDNF1D_IMM:
4260 case AArch64::LD2B_IMM:
4261 case AArch64::LD2H_IMM:
4262 case AArch64::LD2W_IMM:
4263 case AArch64::LD2D_IMM:
4264 case AArch64::ST2B_IMM:
4265 case AArch64::ST2H_IMM:
4266 case AArch64::ST2W_IMM:
4267 case AArch64::ST2D_IMM:
4273 case AArch64::LD3B_IMM:
4274 case AArch64::LD3H_IMM:
4275 case AArch64::LD3W_IMM:
4276 case AArch64::LD3D_IMM:
4277 case AArch64::ST3B_IMM:
4278 case AArch64::ST3H_IMM:
4279 case AArch64::ST3W_IMM:
4280 case AArch64::ST3D_IMM:
4286 case AArch64::LD4B_IMM:
4287 case AArch64::LD4H_IMM:
4288 case AArch64::LD4W_IMM:
4289 case AArch64::LD4D_IMM:
4290 case AArch64::ST4B_IMM:
4291 case AArch64::ST4H_IMM:
4292 case AArch64::ST4W_IMM:
4293 case AArch64::ST4D_IMM:
4299 case AArch64::LD1B_H_IMM:
4300 case AArch64::LD1SB_H_IMM:
4301 case AArch64::LD1H_S_IMM:
4302 case AArch64::LD1SH_S_IMM:
4303 case AArch64::LD1W_D_IMM:
4304 case AArch64::LD1SW_D_IMM:
4305 case AArch64::ST1B_H_IMM:
4306 case AArch64::ST1H_S_IMM:
4307 case AArch64::ST1W_D_IMM:
4308 case AArch64::LDNF1B_H_IMM:
4309 case AArch64::LDNF1SB_H_IMM:
4310 case AArch64::LDNF1H_S_IMM:
4311 case AArch64::LDNF1SH_S_IMM:
4312 case AArch64::LDNF1W_D_IMM:
4313 case AArch64::LDNF1SW_D_IMM:
4321 case AArch64::LD1B_S_IMM:
4322 case AArch64::LD1SB_S_IMM:
4323 case AArch64::LD1H_D_IMM:
4324 case AArch64::LD1SH_D_IMM:
4325 case AArch64::ST1B_S_IMM:
4326 case AArch64::ST1H_D_IMM:
4327 case AArch64::LDNF1B_S_IMM:
4328 case AArch64::LDNF1SB_S_IMM:
4329 case AArch64::LDNF1H_D_IMM:
4330 case AArch64::LDNF1SH_D_IMM:
4338 case AArch64::LD1B_D_IMM:
4339 case AArch64::LD1SB_D_IMM:
4340 case AArch64::ST1B_D_IMM:
4341 case AArch64::LDNF1B_D_IMM:
4342 case AArch64::LDNF1SB_D_IMM:
4350 case AArch64::ST2Gi:
4351 case AArch64::ST2GPreIndex:
4352 case AArch64::ST2GPostIndex:
4353 case AArch64::STZ2Gi:
4354 case AArch64::STZ2GPreIndex:
4355 case AArch64::STZ2GPostIndex:
4361 case AArch64::STGPi:
4362 case AArch64::STGPpost:
4363 case AArch64::STGPpre:
4369 case AArch64::LD1RB_IMM:
4370 case AArch64::LD1RB_H_IMM:
4371 case AArch64::LD1RB_S_IMM:
4372 case AArch64::LD1RB_D_IMM:
4373 case AArch64::LD1RSB_H_IMM:
4374 case AArch64::LD1RSB_S_IMM:
4375 case AArch64::LD1RSB_D_IMM:
4381 case AArch64::LD1RH_IMM:
4382 case AArch64::LD1RH_S_IMM:
4383 case AArch64::LD1RH_D_IMM:
4384 case AArch64::LD1RSH_S_IMM:
4385 case AArch64::LD1RSH_D_IMM:
4391 case AArch64::LD1RW_IMM:
4392 case AArch64::LD1RW_D_IMM:
4393 case AArch64::LD1RSW_IMM:
4399 case AArch64::LD1RD_IMM:
4415 case AArch64::LDRBBui:
4416 case AArch64::LDURBBi:
4417 case AArch64::LDRSBWui:
4418 case AArch64::LDURSBWi:
4419 case AArch64::STRBBui:
4420 case AArch64::STURBBi:
4422 case AArch64::LDRHHui:
4423 case AArch64::LDURHHi:
4424 case AArch64::LDRSHWui:
4425 case AArch64::LDURSHWi:
4426 case AArch64::STRHHui:
4427 case AArch64::STURHHi:
4429 case AArch64::LDRSui:
4430 case AArch64::LDURSi:
4431 case AArch64::LDRSpre:
4432 case AArch64::LDRSWui:
4433 case AArch64::LDURSWi:
4434 case AArch64::LDRSWpre:
4435 case AArch64::LDRWpre:
4436 case AArch64::LDRWui:
4437 case AArch64::LDURWi:
4438 case AArch64::STRSui:
4439 case AArch64::STURSi:
4440 case AArch64::STRSpre:
4441 case AArch64::STRWui:
4442 case AArch64::STURWi:
4443 case AArch64::STRWpre:
4444 case AArch64::LDPSi:
4445 case AArch64::LDPSWi:
4446 case AArch64::LDPWi:
4447 case AArch64::STPSi:
4448 case AArch64::STPWi:
4450 case AArch64::LDRDui:
4451 case AArch64::LDURDi:
4452 case AArch64::LDRDpre:
4453 case AArch64::LDRXui:
4454 case AArch64::LDURXi:
4455 case AArch64::LDRXpre:
4456 case AArch64::STRDui:
4457 case AArch64::STURDi:
4458 case AArch64::STRDpre:
4459 case AArch64::STRXui:
4460 case AArch64::STURXi:
4461 case AArch64::STRXpre:
4462 case AArch64::LDPDi:
4463 case AArch64::LDPXi:
4464 case AArch64::STPDi:
4465 case AArch64::STPXi:
4467 case AArch64::LDRQui:
4468 case AArch64::LDURQi:
4469 case AArch64::STRQui:
4470 case AArch64::STURQi:
4471 case AArch64::STRQpre:
4472 case AArch64::LDPQi:
4473 case AArch64::LDRQpre:
4474 case AArch64::STPQi:
4476 case AArch64::STZGi:
4477 case AArch64::ST2Gi:
4478 case AArch64::STZ2Gi:
4479 case AArch64::STGPi:
4485 switch (
MI.getOpcode()) {
4488 case AArch64::LDRWpre:
4489 case AArch64::LDRXpre:
4490 case AArch64::LDRSWpre:
4491 case AArch64::LDRSpre:
4492 case AArch64::LDRDpre:
4493 case AArch64::LDRQpre:
4499 switch (
MI.getOpcode()) {
4502 case AArch64::STRWpre:
4503 case AArch64::STRXpre:
4504 case AArch64::STRSpre:
4505 case AArch64::STRDpre:
4506 case AArch64::STRQpre:
4516 switch (
MI.getOpcode()) {
4519 case AArch64::LDPSi:
4520 case AArch64::LDPSWi:
4521 case AArch64::LDPDi:
4522 case AArch64::LDPQi:
4523 case AArch64::LDPWi:
4524 case AArch64::LDPXi:
4525 case AArch64::STPSi:
4526 case AArch64::STPDi:
4527 case AArch64::STPQi:
4528 case AArch64::STPWi:
4529 case AArch64::STPXi:
4530 case AArch64::STGPi:
4536 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4540 return MI.getOperand(
Idx);
4545 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4549 return MI.getOperand(
Idx);
4554 switch (
MI.getOpcode()) {
4557 case AArch64::LDRBroX:
4558 case AArch64::LDRBBroX:
4559 case AArch64::LDRSBXroX:
4560 case AArch64::LDRSBWroX:
4561 case AArch64::LDRHroX:
4562 case AArch64::LDRHHroX:
4563 case AArch64::LDRSHXroX:
4564 case AArch64::LDRSHWroX:
4565 case AArch64::LDRWroX:
4566 case AArch64::LDRSroX:
4567 case AArch64::LDRSWroX:
4568 case AArch64::LDRDroX:
4569 case AArch64::LDRXroX:
4570 case AArch64::LDRQroX:
4571 return MI.getOperand(4);
4577 if (
MI.getParent() ==
nullptr)
4587 auto Reg =
Op.getReg();
4588 if (Reg.isPhysical())
4589 return AArch64::FPR16RegClass.
contains(Reg);
4591 return TRC == &AArch64::FPR16RegClass ||
4592 TRC == &AArch64::FPR16_loRegClass;
4601 auto Reg =
Op.getReg();
4602 if (Reg.isPhysical())
4603 return AArch64::FPR128RegClass.
contains(Reg);
4605 return TRC == &AArch64::FPR128RegClass ||
4606 TRC == &AArch64::FPR128_loRegClass;
4612 switch (
MI.getOpcode()) {
4615 case AArch64::PACIASP:
4616 case AArch64::PACIBSP:
4619 case AArch64::PAUTH_PROLOGUE:
4622 case AArch64::HINT: {
4623 unsigned Imm =
MI.getOperand(0).getImm();
4625 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4628 if (Imm == 25 || Imm == 27)
4640 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4641 return AArch64::FPR128RegClass.contains(Reg) ||
4642 AArch64::FPR64RegClass.contains(Reg) ||
4643 AArch64::FPR32RegClass.contains(Reg) ||
4644 AArch64::FPR16RegClass.contains(Reg) ||
4645 AArch64::FPR8RegClass.contains(Reg);
4652 auto Reg =
Op.getReg();
4653 if (Reg.isPhysical())
4657 return TRC == &AArch64::FPR128RegClass ||
4658 TRC == &AArch64::FPR128_loRegClass ||
4659 TRC == &AArch64::FPR64RegClass ||
4660 TRC == &AArch64::FPR64_loRegClass ||
4661 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4662 TRC == &AArch64::FPR8RegClass;
4684 if (FirstOpc == SecondOpc)
4690 case AArch64::STRSui:
4691 case AArch64::STURSi:
4692 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4693 case AArch64::STRDui:
4694 case AArch64::STURDi:
4695 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4696 case AArch64::STRQui:
4697 case AArch64::STURQi:
4698 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4699 case AArch64::STRWui:
4700 case AArch64::STURWi:
4701 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4702 case AArch64::STRXui:
4703 case AArch64::STURXi:
4704 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4705 case AArch64::LDRSui:
4706 case AArch64::LDURSi:
4707 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4708 case AArch64::LDRDui:
4709 case AArch64::LDURDi:
4710 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4711 case AArch64::LDRQui:
4712 case AArch64::LDURQi:
4713 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4714 case AArch64::LDRWui:
4715 case AArch64::LDURWi:
4716 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4717 case AArch64::LDRSWui:
4718 case AArch64::LDURSWi:
4719 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4720 case AArch64::LDRXui:
4721 case AArch64::LDURXi:
4722 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4729 int64_t Offset1,
unsigned Opcode1,
int FI2,
4730 int64_t Offset2,
unsigned Opcode2) {
4736 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4739 if (ObjectOffset1 % Scale1 != 0)
4741 ObjectOffset1 /= Scale1;
4743 if (ObjectOffset2 % Scale2 != 0)
4745 ObjectOffset2 /= Scale2;
4746 ObjectOffset1 += Offset1;
4747 ObjectOffset2 += Offset2;
4748 return ObjectOffset1 + 1 == ObjectOffset2;
4760 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4761 unsigned NumBytes)
const {
4771 "Only base registers and frame indices are supported.");
4778 if (ClusterSize > 2)
4785 unsigned FirstOpc = FirstLdSt.
getOpcode();
4786 unsigned SecondOpc = SecondLdSt.
getOpcode();
4806 if (Offset1 > 63 || Offset1 < -64)
4811 if (BaseOp1.
isFI()) {
4813 "Caller should have ordered offsets.");
4818 BaseOp2.
getIndex(), Offset2, SecondOpc);
4821 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4823 return Offset1 + 1 == Offset2;
4831 return MIB.
addReg(Reg, State);
4834 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4835 return MIB.
addReg(Reg, State, SubIdx);
4842 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4851 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4853 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4854 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4855 unsigned NumRegs = Indices.
size();
4857 int SubReg = 0,
End = NumRegs, Incr = 1;
4876 unsigned Opcode,
unsigned ZeroReg,
4879 unsigned NumRegs = Indices.
size();
4882 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4883 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4884 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4885 "GPR reg sequences should not be able to overlap");
4902 bool RenamableSrc)
const {
4903 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4904 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4907 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4909 if (Subtarget.hasZeroCycleRegMove()) {
4912 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4914 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4930 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4935 if (Subtarget.hasZeroCycleRegMove()) {
4938 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4940 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4960 if (AArch64::PPRRegClass.
contains(DestReg) &&
4961 AArch64::PPRRegClass.
contains(SrcReg)) {
4963 "Unexpected SVE register.");
4973 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4974 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4975 if (DestIsPNR || SrcIsPNR) {
4977 return (R - AArch64::PN0) + AArch64::P0;
4979 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4980 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4982 if (PPRSrcReg != PPRDestReg) {
4994 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4995 AArch64::ZPRRegClass.
contains(SrcReg)) {
4997 "Unexpected SVE register.");
5005 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
5006 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
5007 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
5008 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
5010 "Unexpected SVE register.");
5011 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5018 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
5019 AArch64::ZPR3RegClass.
contains(SrcReg)) {
5021 "Unexpected SVE register.");
5022 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5030 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
5031 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
5032 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
5033 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5035 "Unexpected SVE register.");
5036 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5037 AArch64::zsub2, AArch64::zsub3};
5043 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5044 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
5045 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5051 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
5065 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5066 AArch64::DDDDRegClass.
contains(SrcReg)) {
5067 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5068 AArch64::dsub2, AArch64::dsub3};
5075 if (AArch64::DDDRegClass.
contains(DestReg) &&
5076 AArch64::DDDRegClass.
contains(SrcReg)) {
5077 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5085 if (AArch64::DDRegClass.
contains(DestReg) &&
5086 AArch64::DDRegClass.
contains(SrcReg)) {
5087 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5094 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5095 AArch64::QQQQRegClass.
contains(SrcReg)) {
5096 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5097 AArch64::qsub2, AArch64::qsub3};
5104 if (AArch64::QQQRegClass.
contains(DestReg) &&
5105 AArch64::QQQRegClass.
contains(SrcReg)) {
5106 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5114 if (AArch64::QQRegClass.
contains(DestReg) &&
5115 AArch64::QQRegClass.
contains(SrcReg)) {
5116 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5122 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5123 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5124 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5126 AArch64::XZR, Indices);
5130 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5131 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5132 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5134 AArch64::WZR, Indices);
5138 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5139 AArch64::FPR128RegClass.
contains(SrcReg)) {
5144 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5145 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5165 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5166 AArch64::FPR64RegClass.
contains(SrcReg)) {
5172 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5173 AArch64::FPR32RegClass.
contains(SrcReg)) {
5179 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5180 AArch64::FPR16RegClass.
contains(SrcReg)) {
5182 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
5184 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
5190 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5191 AArch64::FPR8RegClass.
contains(SrcReg)) {
5193 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
5195 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
5202 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5203 AArch64::GPR64RegClass.
contains(SrcReg)) {
5208 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5209 AArch64::FPR64RegClass.
contains(SrcReg)) {
5215 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5216 AArch64::GPR32RegClass.
contains(SrcReg)) {
5221 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5222 AArch64::FPR32RegClass.
contains(SrcReg)) {
5228 if (DestReg == AArch64::NZCV) {
5229 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5231 .
addImm(AArch64SysReg::NZCV)