63#define GET_INSTRINFO_CTOR_DTOR
64#include "AArch64GenInstrInfo.inc"
68 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
72 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
76 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
80 cl::desc(
"Restrict range of B instructions (DEBUG)"));
85 RI(STI.getTargetTriple()), Subtarget(STI) {}
96 auto Op =
MI.getOpcode();
97 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
98 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
102 if (
MI.isMetaInstruction())
107 unsigned NumBytes = 0;
113 switch (
Desc.getOpcode()) {
116 return Desc.getSize();
123 case TargetOpcode::STACKMAP:
126 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
128 case TargetOpcode::PATCHPOINT:
131 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
133 case TargetOpcode::STATEPOINT:
135 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
140 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
145 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
147 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
148 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
152 case TargetOpcode::PATCHABLE_EVENT_CALL:
158 NumBytes =
MI.getOperand(1).getImm();
160 case TargetOpcode::BUNDLE:
161 NumBytes = getInstBundleLength(
MI);
168unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
172 while (++
I != E &&
I->isInsideBundle()) {
173 assert(!
I->isBundle() &&
"No nested bundle!");
232 int64_t BrOffset)
const {
234 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
235 "over conditional branch expansion");
236 return isIntN(Bits, BrOffset / 4);
241 switch (
MI.getOpcode()) {
245 return MI.getOperand(0).getMBB();
250 return MI.getOperand(2).getMBB();
256 return MI.getOperand(1).getMBB();
266 assert(RS &&
"RegScavenger required for long branching");
268 "new block should be inserted for expanding unconditional branch");
271 "restore block should be inserted for restoring clobbered registers");
276 if (!isInt<33>(BrOffset))
278 "Branch offsets outside of the signed 33-bit range not supported");
292 constexpr Register Reg = AArch64::X16;
294 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
302 if (Scavenged != AArch64::NoRegister &&
304 buildIndirectBranch(Scavenged, NewDestBB);
314 "Unable to insert indirect branch inside function that has red zone");
337 bool AllowModify)
const {
344 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
345 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
349 if (!isUnpredicatedTerminator(*
I))
356 unsigned LastOpc = LastInst->
getOpcode();
357 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
372 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
379 LastInst = SecondLastInst;
381 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
386 SecondLastInst = &*
I;
387 SecondLastOpc = SecondLastInst->
getOpcode();
398 LastInst = SecondLastInst;
400 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
402 "unreachable unconditional branches removed above");
411 SecondLastInst = &*
I;
412 SecondLastOpc = SecondLastInst->
getOpcode();
416 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
432 I->eraseFromParent();
441 I->eraseFromParent();
450 MachineBranchPredicate &MBP,
451 bool AllowModify)
const {
461 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
462 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
466 if (!isUnpredicatedTerminator(*
I))
471 unsigned LastOpc = LastInst->
getOpcode();
486 assert(MBP.TrueDest &&
"expected!");
489 MBP.ConditionDef =
nullptr;
490 MBP.SingleUseCondition =
false;
494 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
495 : MachineBranchPredicate::PRED_EQ;
501 if (
Cond[0].getImm() != -1) {
507 switch (
Cond[1].getImm()) {
511 Cond[1].setImm(AArch64::CBNZW);
514 Cond[1].setImm(AArch64::CBZW);
517 Cond[1].setImm(AArch64::CBNZX);
520 Cond[1].setImm(AArch64::CBZX);
523 Cond[1].setImm(AArch64::TBNZW);
526 Cond[1].setImm(AArch64::TBZW);
529 Cond[1].setImm(AArch64::TBNZX);
532 Cond[1].setImm(AArch64::TBZX);
541 int *BytesRemoved)
const {
551 I->eraseFromParent();
568 I->eraseFromParent();
575void AArch64InstrInfo::instantiateCondBranch(
578 if (
Cond[0].getImm() != -1) {
596 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
635 unsigned *NewVReg =
nullptr) {
640 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
643 unsigned SrcOpNum = 0;
645 case AArch64::ADDSXri:
646 case AArch64::ADDSWri:
653 case AArch64::ADDXri:
654 case AArch64::ADDWri:
660 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
663 case AArch64::ORNXrr:
664 case AArch64::ORNWrr: {
667 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
670 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
674 case AArch64::SUBSXrr:
675 case AArch64::SUBSWrr:
682 case AArch64::SUBXrr:
683 case AArch64::SUBWrr: {
686 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
689 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
695 assert(Opc && SrcOpNum &&
"Missing parameters");
707 int &FalseCycles)
const {
711 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
718 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
722 unsigned ExtraCondLat =
Cond.size() != 1;
726 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
727 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
729 CondCycles = 1 + ExtraCondLat;
730 TrueCycles = FalseCycles = 1;
740 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
741 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
742 CondCycles = 5 + ExtraCondLat;
743 TrueCycles = FalseCycles = 2;
760 switch (
Cond.size()) {
769 switch (
Cond[1].getImm()) {
792 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
798 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
808 switch (
Cond[1].getImm()) {
821 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
837 bool TryFold =
false;
838 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
839 RC = &AArch64::GPR64RegClass;
840 Opc = AArch64::CSELXr;
842 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
843 RC = &AArch64::GPR32RegClass;
844 Opc = AArch64::CSELWr;
846 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
847 RC = &AArch64::FPR64RegClass;
848 Opc = AArch64::FCSELDrrr;
849 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
850 RC = &AArch64::FPR32RegClass;
851 Opc = AArch64::FCSELSrrr;
853 assert(RC &&
"Unsupported regclass");
857 unsigned NewVReg = 0;
872 MRI.clearKillFlags(NewVReg);
877 MRI.constrainRegClass(TrueReg, RC);
878 MRI.constrainRegClass(FalseReg, RC);
893 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
898 return Is.
size() <= 2;
904 if (Subtarget.hasExynosCheapAsMoveHandling()) {
905 if (isExynosCheapAsMove(
MI))
907 return MI.isAsCheapAsAMove();
910 switch (
MI.getOpcode()) {
912 return MI.isAsCheapAsAMove();
914 case AArch64::ADDWrs:
915 case AArch64::ADDXrs:
916 case AArch64::SUBWrs:
917 case AArch64::SUBXrs:
918 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
923 case AArch64::MOVi32imm:
925 case AArch64::MOVi64imm:
931 switch (
MI.getOpcode()) {
935 case AArch64::ADDWrs:
936 case AArch64::ADDXrs:
937 case AArch64::ADDSWrs:
938 case AArch64::ADDSXrs: {
939 unsigned Imm =
MI.getOperand(3).getImm();
946 case AArch64::ADDWrx:
947 case AArch64::ADDXrx:
948 case AArch64::ADDXrx64:
949 case AArch64::ADDSWrx:
950 case AArch64::ADDSXrx:
951 case AArch64::ADDSXrx64: {
952 unsigned Imm =
MI.getOperand(3).getImm();
964 case AArch64::SUBWrs:
965 case AArch64::SUBSWrs: {
966 unsigned Imm =
MI.getOperand(3).getImm();
968 return ShiftVal == 0 ||
972 case AArch64::SUBXrs:
973 case AArch64::SUBSXrs: {
974 unsigned Imm =
MI.getOperand(3).getImm();
976 return ShiftVal == 0 ||
980 case AArch64::SUBWrx:
981 case AArch64::SUBXrx:
982 case AArch64::SUBXrx64:
983 case AArch64::SUBSWrx:
984 case AArch64::SUBSXrx:
985 case AArch64::SUBSXrx64: {
986 unsigned Imm =
MI.getOperand(3).getImm();
998 case AArch64::LDRBBroW:
999 case AArch64::LDRBBroX:
1000 case AArch64::LDRBroW:
1001 case AArch64::LDRBroX:
1002 case AArch64::LDRDroW:
1003 case AArch64::LDRDroX:
1004 case AArch64::LDRHHroW:
1005 case AArch64::LDRHHroX:
1006 case AArch64::LDRHroW:
1007 case AArch64::LDRHroX:
1008 case AArch64::LDRQroW:
1009 case AArch64::LDRQroX:
1010 case AArch64::LDRSBWroW:
1011 case AArch64::LDRSBWroX:
1012 case AArch64::LDRSBXroW:
1013 case AArch64::LDRSBXroX:
1014 case AArch64::LDRSHWroW:
1015 case AArch64::LDRSHWroX:
1016 case AArch64::LDRSHXroW:
1017 case AArch64::LDRSHXroX:
1018 case AArch64::LDRSWroW:
1019 case AArch64::LDRSWroX:
1020 case AArch64::LDRSroW:
1021 case AArch64::LDRSroX:
1022 case AArch64::LDRWroW:
1023 case AArch64::LDRWroX:
1024 case AArch64::LDRXroW:
1025 case AArch64::LDRXroX:
1026 case AArch64::PRFMroW:
1027 case AArch64::PRFMroX:
1028 case AArch64::STRBBroW:
1029 case AArch64::STRBBroX:
1030 case AArch64::STRBroW:
1031 case AArch64::STRBroX:
1032 case AArch64::STRDroW:
1033 case AArch64::STRDroX:
1034 case AArch64::STRHHroW:
1035 case AArch64::STRHHroX:
1036 case AArch64::STRHroW:
1037 case AArch64::STRHroX:
1038 case AArch64::STRQroW:
1039 case AArch64::STRQroX:
1040 case AArch64::STRSroW:
1041 case AArch64::STRSroX:
1042 case AArch64::STRWroW:
1043 case AArch64::STRWroX:
1044 case AArch64::STRXroW:
1045 case AArch64::STRXroX: {
1046 unsigned IsSigned =
MI.getOperand(3).getImm();
1053 unsigned Opc =
MI.getOpcode();
1057 case AArch64::SEH_StackAlloc:
1058 case AArch64::SEH_SaveFPLR:
1059 case AArch64::SEH_SaveFPLR_X:
1060 case AArch64::SEH_SaveReg:
1061 case AArch64::SEH_SaveReg_X:
1062 case AArch64::SEH_SaveRegP:
1063 case AArch64::SEH_SaveRegP_X:
1064 case AArch64::SEH_SaveFReg:
1065 case AArch64::SEH_SaveFReg_X:
1066 case AArch64::SEH_SaveFRegP:
1067 case AArch64::SEH_SaveFRegP_X:
1068 case AArch64::SEH_SetFP:
1069 case AArch64::SEH_AddFP:
1070 case AArch64::SEH_Nop:
1071 case AArch64::SEH_PrologEnd:
1072 case AArch64::SEH_EpilogStart:
1073 case AArch64::SEH_EpilogEnd:
1074 case AArch64::SEH_PACSignLR:
1075 case AArch64::SEH_SaveAnyRegQP:
1076 case AArch64::SEH_SaveAnyRegQPX:
1083 unsigned &SubIdx)
const {
1084 switch (
MI.getOpcode()) {
1087 case AArch64::SBFMXri:
1088 case AArch64::UBFMXri:
1091 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1094 SrcReg =
MI.getOperand(1).getReg();
1095 DstReg =
MI.getOperand(0).getReg();
1096 SubIdx = AArch64::sub_32;
1105 int64_t OffsetA = 0, OffsetB = 0;
1106 TypeSize WidthA(0,
false), WidthB(0,
false);
1107 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1128 OffsetAIsScalable == OffsetBIsScalable) {
1129 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1130 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1131 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1132 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1150 switch (
MI.getOpcode()) {
1153 if (
MI.getOperand(0).getImm() == 0x14)
1160 case AArch64::MSRpstatesvcrImm1:
1167 auto Next = std::next(
MI.getIterator());
1168 return Next !=
MBB->
end() && Next->isCFIInstruction();
1175 Register &SrcReg2, int64_t &CmpMask,
1176 int64_t &CmpValue)
const {
1179 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1180 if (!
MI.getOperand(1).isReg())
1183 switch (
MI.getOpcode()) {
1186 case AArch64::PTEST_PP:
1187 case AArch64::PTEST_PP_ANY:
1188 SrcReg =
MI.getOperand(0).getReg();
1189 SrcReg2 =
MI.getOperand(1).getReg();
1194 case AArch64::SUBSWrr:
1195 case AArch64::SUBSWrs:
1196 case AArch64::SUBSWrx:
1197 case AArch64::SUBSXrr:
1198 case AArch64::SUBSXrs:
1199 case AArch64::SUBSXrx:
1200 case AArch64::ADDSWrr:
1201 case AArch64::ADDSWrs:
1202 case AArch64::ADDSWrx:
1203 case AArch64::ADDSXrr:
1204 case AArch64::ADDSXrs:
1205 case AArch64::ADDSXrx:
1207 SrcReg =
MI.getOperand(1).getReg();
1208 SrcReg2 =
MI.getOperand(2).getReg();
1212 case AArch64::SUBSWri:
1213 case AArch64::ADDSWri:
1214 case AArch64::SUBSXri:
1215 case AArch64::ADDSXri:
1216 SrcReg =
MI.getOperand(1).getReg();
1219 CmpValue =
MI.getOperand(2).getImm();
1221 case AArch64::ANDSWri:
1222 case AArch64::ANDSXri:
1225 SrcReg =
MI.getOperand(1).getReg();
1229 MI.getOperand(2).getImm(),
1230 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1239 assert(
MBB &&
"Can't get MachineBasicBlock here");
1241 assert(MF &&
"Can't get MachineFunction here");
1246 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1250 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1253 if (!OpRegCstraints)
1261 "Operand has register constraints without being a register!");
1264 if (Reg.isPhysical()) {
1265 if (!OpRegCstraints->
contains(Reg))
1268 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1281 bool MIDefinesZeroReg =
false;
1282 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1283 MI.definesRegister(AArch64::XZR,
nullptr))
1284 MIDefinesZeroReg =
true;
1286 switch (
MI.getOpcode()) {
1288 return MI.getOpcode();
1289 case AArch64::ADDSWrr:
1290 return AArch64::ADDWrr;
1291 case AArch64::ADDSWri:
1292 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1293 case AArch64::ADDSWrs:
1294 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1295 case AArch64::ADDSWrx:
1296 return AArch64::ADDWrx;
1297 case AArch64::ADDSXrr:
1298 return AArch64::ADDXrr;
1299 case AArch64::ADDSXri:
1300 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1301 case AArch64::ADDSXrs:
1302 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1303 case AArch64::ADDSXrx:
1304 return AArch64::ADDXrx;
1305 case AArch64::SUBSWrr:
1306 return AArch64::SUBWrr;
1307 case AArch64::SUBSWri:
1308 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1309 case AArch64::SUBSWrs:
1310 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1311 case AArch64::SUBSWrx:
1312 return AArch64::SUBWrx;
1313 case AArch64::SUBSXrr:
1314 return AArch64::SUBXrr;
1315 case AArch64::SUBSXri:
1316 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1317 case AArch64::SUBSXrs:
1318 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1319 case AArch64::SUBSXrx:
1320 return AArch64::SUBXrx;
1335 if (To == To->getParent()->begin())
1340 if (To->getParent() !=
From->getParent())
1352 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1353 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1359std::optional<unsigned>
1363 unsigned MaskOpcode =
Mask->getOpcode();
1364 unsigned PredOpcode = Pred->
getOpcode();
1368 if (PredIsWhileLike) {
1372 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1386 if (PredIsPTestLike) {
1391 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1401 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1430 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1438 switch (PredOpcode) {
1439 case AArch64::AND_PPzPP:
1440 case AArch64::BIC_PPzPP:
1441 case AArch64::EOR_PPzPP:
1442 case AArch64::NAND_PPzPP:
1443 case AArch64::NOR_PPzPP:
1444 case AArch64::ORN_PPzPP:
1445 case AArch64::ORR_PPzPP:
1446 case AArch64::BRKA_PPzP:
1447 case AArch64::BRKPA_PPzPP:
1448 case AArch64::BRKB_PPzP:
1449 case AArch64::BRKPB_PPzPP:
1450 case AArch64::RDFFR_PPz: {
1454 if (Mask != PredMask)
1458 case AArch64::BRKN_PPzP: {
1462 if ((MaskOpcode != AArch64::PTRUE_B) ||
1463 (
Mask->getOperand(1).getImm() != 31))
1467 case AArch64::PTRUE_B:
1480bool AArch64InstrInfo::optimizePTestInstr(
1481 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1483 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1484 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1485 unsigned PredOpcode = Pred->
getOpcode();
1486 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1502 if (*NewOp != PredOpcode) {
1513 for (; i !=
e; ++i) {
1544 if (DeadNZCVIdx != -1) {
1563 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1564 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1565 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1574 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1576 return (CmpValue == 0 || CmpValue == 1) &&
1577 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1585 switch (Instr.getOpcode()) {
1587 return AArch64::INSTRUCTION_LIST_END;
1589 case AArch64::ADDSWrr:
1590 case AArch64::ADDSWri:
1591 case AArch64::ADDSXrr:
1592 case AArch64::ADDSXri:
1593 case AArch64::SUBSWrr:
1594 case AArch64::SUBSWri:
1595 case AArch64::SUBSXrr:
1596 case AArch64::SUBSXri:
1597 return Instr.getOpcode();
1599 case AArch64::ADDWrr:
1600 return AArch64::ADDSWrr;
1601 case AArch64::ADDWri:
1602 return AArch64::ADDSWri;
1603 case AArch64::ADDXrr:
1604 return AArch64::ADDSXrr;
1605 case AArch64::ADDXri:
1606 return AArch64::ADDSXri;
1607 case AArch64::ADCWr:
1608 return AArch64::ADCSWr;
1609 case AArch64::ADCXr:
1610 return AArch64::ADCSXr;
1611 case AArch64::SUBWrr:
1612 return AArch64::SUBSWrr;
1613 case AArch64::SUBWri:
1614 return AArch64::SUBSWri;
1615 case AArch64::SUBXrr:
1616 return AArch64::SUBSXrr;
1617 case AArch64::SUBXri:
1618 return AArch64::SUBSXri;
1619 case AArch64::SBCWr:
1620 return AArch64::SBCSWr;
1621 case AArch64::SBCXr:
1622 return AArch64::SBCSXr;
1623 case AArch64::ANDWri:
1624 return AArch64::ANDSWri;
1625 case AArch64::ANDXri:
1626 return AArch64::ANDSXri;
1633 if (BB->isLiveIn(AArch64::NZCV))
1642 switch (Instr.getOpcode()) {
1646 case AArch64::Bcc: {
1647 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1652 case AArch64::CSINVWr:
1653 case AArch64::CSINVXr:
1654 case AArch64::CSINCWr:
1655 case AArch64::CSINCXr:
1656 case AArch64::CSELWr:
1657 case AArch64::CSELXr:
1658 case AArch64::CSNEGWr:
1659 case AArch64::CSNEGXr:
1660 case AArch64::FCSELSrrr:
1661 case AArch64::FCSELDrrr: {
1662 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1675 Instr.getOperand(CCIdx).getImm())
1728std::optional<UsedNZCV>
1733 if (
MI.getParent() != CmpParent)
1734 return std::nullopt;
1737 return std::nullopt;
1742 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1745 return std::nullopt;
1750 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1753 return NZCVUsedAfterCmp;
1757 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1761 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1783 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1789 "Caller guarantees that CmpInstr compares with constant 0");
1792 if (!NZVCUsed || NZVCUsed->C)
1814bool AArch64InstrInfo::substituteCmpToZero(
1825 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1832 MI->setDesc(
get(NewOpc));
1837 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1849 assert((CmpValue == 0 || CmpValue == 1) &&
1850 "Only comparisons to 0 or 1 considered for removal!");
1853 unsigned MIOpc =
MI.getOpcode();
1854 if (MIOpc == AArch64::CSINCWr) {
1855 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1856 MI.getOperand(2).getReg() != AArch64::WZR)
1858 }
else if (MIOpc == AArch64::CSINCXr) {
1859 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1860 MI.getOperand(2).getReg() != AArch64::XZR)
1870 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1874 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1876 if (CmpValue && !IsSubsRegImm)
1878 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1883 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1886 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1890 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1893 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1894 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1897 if (MIUsedNZCV.
N && !CmpValue)
1939bool AArch64InstrInfo::removeCmpToZeroOrOne(
1947 bool IsInvertCC =
false;
1957 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1968 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1969 MI.getOpcode() != AArch64::CATCHRET)
1977 if (
MI.getOpcode() == AArch64::CATCHRET) {
1986 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1988 FirstEpilogSEH = std::next(FirstEpilogSEH);
2002 if (M.getStackProtectorGuard() ==
"sysreg") {
2012 int Offset = M.getStackProtectorGuardOffset();
2064 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2073 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2115 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2138 switch (
MI.getOpcode()) {
2141 case AArch64::MOVZWi:
2142 case AArch64::MOVZXi:
2143 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2144 assert(
MI.getDesc().getNumOperands() == 3 &&
2145 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2149 case AArch64::ANDWri:
2150 return MI.getOperand(1).getReg() == AArch64::WZR;
2151 case AArch64::ANDXri:
2152 return MI.getOperand(1).getReg() == AArch64::XZR;
2153 case TargetOpcode::COPY:
2154 return MI.getOperand(1).getReg() == AArch64::WZR;
2162 switch (
MI.getOpcode()) {
2165 case TargetOpcode::COPY: {
2168 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2169 AArch64::GPR64RegClass.
contains(DstReg));
2171 case AArch64::ORRXrs:
2172 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2173 assert(
MI.getDesc().getNumOperands() == 4 &&
2174 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2178 case AArch64::ADDXri:
2179 if (
MI.getOperand(2).getImm() == 0) {
2180 assert(
MI.getDesc().getNumOperands() == 4 &&
2181 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2192 switch (
MI.getOpcode()) {
2195 case TargetOpcode::COPY: {
2197 return AArch64::FPR128RegClass.contains(DstReg);
2199 case AArch64::ORRv16i8:
2200 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2201 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2202 "invalid ORRv16i8 operands");
2211 int &FrameIndex)
const {
2212 switch (
MI.getOpcode()) {
2215 case AArch64::LDRWui:
2216 case AArch64::LDRXui:
2217 case AArch64::LDRBui:
2218 case AArch64::LDRHui:
2219 case AArch64::LDRSui:
2220 case AArch64::LDRDui:
2221 case AArch64::LDRQui:
2222 case AArch64::LDR_PXI:
2223 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2224 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2225 FrameIndex =
MI.getOperand(1).getIndex();
2226 return MI.getOperand(0).getReg();
2235 int &FrameIndex)
const {
2236 switch (
MI.getOpcode()) {
2239 case AArch64::STRWui:
2240 case AArch64::STRXui:
2241 case AArch64::STRBui:
2242 case AArch64::STRHui:
2243 case AArch64::STRSui:
2244 case AArch64::STRDui:
2245 case AArch64::STRQui:
2246 case AArch64::STR_PXI:
2247 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2248 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2249 FrameIndex =
MI.getOperand(1).getIndex();
2250 return MI.getOperand(0).getReg();
2260 return MMO->getFlags() & MOSuppressPair;
2266 if (
MI.memoperands_empty())
2274 return MMO->getFlags() & MOStridedAccess;
2282 case AArch64::STURSi:
2283 case AArch64::STRSpre:
2284 case AArch64::STURDi:
2285 case AArch64::STRDpre:
2286 case AArch64::STURQi:
2287 case AArch64::STRQpre:
2288 case AArch64::STURBBi:
2289 case AArch64::STURHHi:
2290 case AArch64::STURWi:
2291 case AArch64::STRWpre:
2292 case AArch64::STURXi:
2293 case AArch64::STRXpre:
2294 case AArch64::LDURSi:
2295 case AArch64::LDRSpre:
2296 case AArch64::LDURDi:
2297 case AArch64::LDRDpre:
2298 case AArch64::LDURQi:
2299 case AArch64::LDRQpre:
2300 case AArch64::LDURWi:
2301 case AArch64::LDRWpre:
2302 case AArch64::LDURXi:
2303 case AArch64::LDRXpre:
2304 case AArch64::LDRSWpre:
2305 case AArch64::LDURSWi:
2306 case AArch64::LDURHHi:
2307 case AArch64::LDURBBi:
2308 case AArch64::LDURSBWi:
2309 case AArch64::LDURSHWi:
2317 case AArch64::PRFMui:
return AArch64::PRFUMi;
2318 case AArch64::LDRXui:
return AArch64::LDURXi;
2319 case AArch64::LDRWui:
return AArch64::LDURWi;
2320 case AArch64::LDRBui:
return AArch64::LDURBi;
2321 case AArch64::LDRHui:
return AArch64::LDURHi;
2322 case AArch64::LDRSui:
return AArch64::LDURSi;
2323 case AArch64::LDRDui:
return AArch64::LDURDi;
2324 case AArch64::LDRQui:
return AArch64::LDURQi;
2325 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2326 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2327 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2328 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2329 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2330 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2331 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2332 case AArch64::STRXui:
return AArch64::STURXi;
2333 case AArch64::STRWui:
return AArch64::STURWi;
2334 case AArch64::STRBui:
return AArch64::STURBi;
2335 case AArch64::STRHui:
return AArch64::STURHi;
2336 case AArch64::STRSui:
return AArch64::STURSi;
2337 case AArch64::STRDui:
return AArch64::STURDi;
2338 case AArch64::STRQui:
return AArch64::STURQi;
2339 case AArch64::STRBBui:
return AArch64::STURBBi;
2340 case AArch64::STRHHui:
return AArch64::STURHHi;
2349 case AArch64::LDAPURBi:
2350 case AArch64::LDAPURHi:
2351 case AArch64::LDAPURi:
2352 case AArch64::LDAPURSBWi:
2353 case AArch64::LDAPURSBXi:
2354 case AArch64::LDAPURSHWi:
2355 case AArch64::LDAPURSHXi:
2356 case AArch64::LDAPURSWi:
2357 case AArch64::LDAPURXi:
2358 case AArch64::LDR_PPXI:
2359 case AArch64::LDR_PXI:
2360 case AArch64::LDR_ZXI:
2361 case AArch64::LDR_ZZXI:
2362 case AArch64::LDR_ZZZXI:
2363 case AArch64::LDR_ZZZZXI:
2364 case AArch64::LDRBBui:
2365 case AArch64::LDRBui:
2366 case AArch64::LDRDui:
2367 case AArch64::LDRHHui:
2368 case AArch64::LDRHui:
2369 case AArch64::LDRQui:
2370 case AArch64::LDRSBWui:
2371 case AArch64::LDRSBXui:
2372 case AArch64::LDRSHWui:
2373 case AArch64::LDRSHXui:
2374 case AArch64::LDRSui:
2375 case AArch64::LDRSWui:
2376 case AArch64::LDRWui:
2377 case AArch64::LDRXui:
2378 case AArch64::LDURBBi:
2379 case AArch64::LDURBi:
2380 case AArch64::LDURDi:
2381 case AArch64::LDURHHi:
2382 case AArch64::LDURHi:
2383 case AArch64::LDURQi:
2384 case AArch64::LDURSBWi:
2385 case AArch64::LDURSBXi:
2386 case AArch64::LDURSHWi:
2387 case AArch64::LDURSHXi:
2388 case AArch64::LDURSi:
2389 case AArch64::LDURSWi:
2390 case AArch64::LDURWi:
2391 case AArch64::LDURXi:
2392 case AArch64::PRFMui:
2393 case AArch64::PRFUMi:
2394 case AArch64::ST2Gi:
2396 case AArch64::STLURBi:
2397 case AArch64::STLURHi:
2398 case AArch64::STLURWi:
2399 case AArch64::STLURXi:
2400 case AArch64::StoreSwiftAsyncContext:
2401 case AArch64::STR_PPXI:
2402 case AArch64::STR_PXI:
2403 case AArch64::STR_ZXI:
2404 case AArch64::STR_ZZXI:
2405 case AArch64::STR_ZZZXI:
2406 case AArch64::STR_ZZZZXI:
2407 case AArch64::STRBBui:
2408 case AArch64::STRBui:
2409 case AArch64::STRDui:
2410 case AArch64::STRHHui:
2411 case AArch64::STRHui:
2412 case AArch64::STRQui:
2413 case AArch64::STRSui:
2414 case AArch64::STRWui:
2415 case AArch64::STRXui:
2416 case AArch64::STURBBi:
2417 case AArch64::STURBi:
2418 case AArch64::STURDi:
2419 case AArch64::STURHHi:
2420 case AArch64::STURHi:
2421 case AArch64::STURQi:
2422 case AArch64::STURSi:
2423 case AArch64::STURWi:
2424 case AArch64::STURXi:
2425 case AArch64::STZ2Gi:
2426 case AArch64::STZGi:
2427 case AArch64::TAGPstack:
2429 case AArch64::LD1B_D_IMM:
2430 case AArch64::LD1B_H_IMM:
2431 case AArch64::LD1B_IMM:
2432 case AArch64::LD1B_S_IMM:
2433 case AArch64::LD1D_IMM:
2434 case AArch64::LD1H_D_IMM:
2435 case AArch64::LD1H_IMM:
2436 case AArch64::LD1H_S_IMM:
2437 case AArch64::LD1RB_D_IMM:
2438 case AArch64::LD1RB_H_IMM:
2439 case AArch64::LD1RB_IMM:
2440 case AArch64::LD1RB_S_IMM:
2441 case AArch64::LD1RD_IMM:
2442 case AArch64::LD1RH_D_IMM:
2443 case AArch64::LD1RH_IMM:
2444 case AArch64::LD1RH_S_IMM:
2445 case AArch64::LD1RSB_D_IMM:
2446 case AArch64::LD1RSB_H_IMM:
2447 case AArch64::LD1RSB_S_IMM:
2448 case AArch64::LD1RSH_D_IMM:
2449 case AArch64::LD1RSH_S_IMM:
2450 case AArch64::LD1RSW_IMM:
2451 case AArch64::LD1RW_D_IMM:
2452 case AArch64::LD1RW_IMM:
2453 case AArch64::LD1SB_D_IMM:
2454 case AArch64::LD1SB_H_IMM:
2455 case AArch64::LD1SB_S_IMM:
2456 case AArch64::LD1SH_D_IMM:
2457 case AArch64::LD1SH_S_IMM:
2458 case AArch64::LD1SW_D_IMM:
2459 case AArch64::LD1W_D_IMM:
2460 case AArch64::LD1W_IMM:
2461 case AArch64::LD2B_IMM:
2462 case AArch64::LD2D_IMM:
2463 case AArch64::LD2H_IMM:
2464 case AArch64::LD2W_IMM:
2465 case AArch64::LD3B_IMM:
2466 case AArch64::LD3D_IMM:
2467 case AArch64::LD3H_IMM:
2468 case AArch64::LD3W_IMM:
2469 case AArch64::LD4B_IMM:
2470 case AArch64::LD4D_IMM:
2471 case AArch64::LD4H_IMM:
2472 case AArch64::LD4W_IMM:
2474 case AArch64::LDNF1B_D_IMM:
2475 case AArch64::LDNF1B_H_IMM:
2476 case AArch64::LDNF1B_IMM:
2477 case AArch64::LDNF1B_S_IMM:
2478 case AArch64::LDNF1D_IMM:
2479 case AArch64::LDNF1H_D_IMM:
2480 case AArch64::LDNF1H_IMM:
2481 case AArch64::LDNF1H_S_IMM:
2482 case AArch64::LDNF1SB_D_IMM:
2483 case AArch64::LDNF1SB_H_IMM:
2484 case AArch64::LDNF1SB_S_IMM:
2485 case AArch64::LDNF1SH_D_IMM:
2486 case AArch64::LDNF1SH_S_IMM:
2487 case AArch64::LDNF1SW_D_IMM:
2488 case AArch64::LDNF1W_D_IMM:
2489 case AArch64::LDNF1W_IMM:
2490 case AArch64::LDNPDi:
2491 case AArch64::LDNPQi:
2492 case AArch64::LDNPSi:
2493 case AArch64::LDNPWi:
2494 case AArch64::LDNPXi:
2495 case AArch64::LDNT1B_ZRI:
2496 case AArch64::LDNT1D_ZRI:
2497 case AArch64::LDNT1H_ZRI:
2498 case AArch64::LDNT1W_ZRI:
2499 case AArch64::LDPDi:
2500 case AArch64::LDPQi:
2501 case AArch64::LDPSi:
2502 case AArch64::LDPWi:
2503 case AArch64::LDPXi:
2504 case AArch64::LDRBBpost:
2505 case AArch64::LDRBBpre:
2506 case AArch64::LDRBpost:
2507 case AArch64::LDRBpre:
2508 case AArch64::LDRDpost:
2509 case AArch64::LDRDpre:
2510 case AArch64::LDRHHpost:
2511 case AArch64::LDRHHpre:
2512 case AArch64::LDRHpost:
2513 case AArch64::LDRHpre:
2514 case AArch64::LDRQpost:
2515 case AArch64::LDRQpre:
2516 case AArch64::LDRSpost:
2517 case AArch64::LDRSpre:
2518 case AArch64::LDRWpost:
2519 case AArch64::LDRWpre:
2520 case AArch64::LDRXpost:
2521 case AArch64::LDRXpre:
2522 case AArch64::ST1B_D_IMM:
2523 case AArch64::ST1B_H_IMM:
2524 case AArch64::ST1B_IMM:
2525 case AArch64::ST1B_S_IMM:
2526 case AArch64::ST1D_IMM:
2527 case AArch64::ST1H_D_IMM:
2528 case AArch64::ST1H_IMM:
2529 case AArch64::ST1H_S_IMM:
2530 case AArch64::ST1W_D_IMM:
2531 case AArch64::ST1W_IMM:
2532 case AArch64::ST2B_IMM:
2533 case AArch64::ST2D_IMM:
2534 case AArch64::ST2H_IMM:
2535 case AArch64::ST2W_IMM:
2536 case AArch64::ST3B_IMM:
2537 case AArch64::ST3D_IMM:
2538 case AArch64::ST3H_IMM:
2539 case AArch64::ST3W_IMM:
2540 case AArch64::ST4B_IMM:
2541 case AArch64::ST4D_IMM:
2542 case AArch64::ST4H_IMM:
2543 case AArch64::ST4W_IMM:
2544 case AArch64::STGPi:
2545 case AArch64::STNPDi:
2546 case AArch64::STNPQi:
2547 case AArch64::STNPSi:
2548 case AArch64::STNPWi:
2549 case AArch64::STNPXi:
2550 case AArch64::STNT1B_ZRI:
2551 case AArch64::STNT1D_ZRI:
2552 case AArch64::STNT1H_ZRI:
2553 case AArch64::STNT1W_ZRI:
2554 case AArch64::STPDi:
2555 case AArch64::STPQi:
2556 case AArch64::STPSi:
2557 case AArch64::STPWi:
2558 case AArch64::STPXi:
2559 case AArch64::STRBBpost:
2560 case AArch64::STRBBpre:
2561 case AArch64::STRBpost:
2562 case AArch64::STRBpre:
2563 case AArch64::STRDpost:
2564 case AArch64::STRDpre:
2565 case AArch64::STRHHpost:
2566 case AArch64::STRHHpre:
2567 case AArch64::STRHpost:
2568 case AArch64::STRHpre:
2569 case AArch64::STRQpost:
2570 case AArch64::STRQpre:
2571 case AArch64::STRSpost:
2572 case AArch64::STRSpre:
2573 case AArch64::STRWpost:
2574 case AArch64::STRWpre:
2575 case AArch64::STRXpost:
2576 case AArch64::STRXpre:
2578 case AArch64::LDPDpost:
2579 case AArch64::LDPDpre:
2580 case AArch64::LDPQpost:
2581 case AArch64::LDPQpre:
2582 case AArch64::LDPSpost:
2583 case AArch64::LDPSpre:
2584 case AArch64::LDPWpost:
2585 case AArch64::LDPWpre:
2586 case AArch64::LDPXpost:
2587 case AArch64::LDPXpre:
2588 case AArch64::STPDpost:
2589 case AArch64::STPDpre:
2590 case AArch64::STPQpost:
2591 case AArch64::STPQpre:
2592 case AArch64::STPSpost:
2593 case AArch64::STPSpre:
2594 case AArch64::STPWpost:
2595 case AArch64::STPWpre:
2596 case AArch64::STPXpost:
2597 case AArch64::STPXpre:
2603 switch (
MI.getOpcode()) {
2607 case AArch64::STRSui:
2608 case AArch64::STRDui:
2609 case AArch64::STRQui:
2610 case AArch64::STRXui:
2611 case AArch64::STRWui:
2612 case AArch64::LDRSui:
2613 case AArch64::LDRDui:
2614 case AArch64::LDRQui:
2615 case AArch64::LDRXui:
2616 case AArch64::LDRWui:
2617 case AArch64::LDRSWui:
2619 case AArch64::STURSi:
2620 case AArch64::STRSpre:
2621 case AArch64::STURDi:
2622 case AArch64::STRDpre:
2623 case AArch64::STURQi:
2624 case AArch64::STRQpre:
2625 case AArch64::STURWi:
2626 case AArch64::STRWpre:
2627 case AArch64::STURXi:
2628 case AArch64::STRXpre:
2629 case AArch64::LDURSi:
2630 case AArch64::LDRSpre:
2631 case AArch64::LDURDi:
2632 case AArch64::LDRDpre:
2633 case AArch64::LDURQi:
2634 case AArch64::LDRQpre:
2635 case AArch64::LDURWi:
2636 case AArch64::LDRWpre:
2637 case AArch64::LDURXi:
2638 case AArch64::LDRXpre:
2639 case AArch64::LDURSWi:
2640 case AArch64::LDRSWpre:
2646 switch (
MI.getOpcode()) {
2649 "Unexpected instruction - was a new tail call opcode introduced?");
2651 case AArch64::TCRETURNdi:
2652 case AArch64::TCRETURNri:
2653 case AArch64::TCRETURNrix16x17:
2654 case AArch64::TCRETURNrix17:
2655 case AArch64::TCRETURNrinotx16:
2656 case AArch64::TCRETURNriALL:
2657 case AArch64::AUTH_TCRETURN:
2658 case AArch64::AUTH_TCRETURN_BTI:
2668 case AArch64::ADDWri:
2669 return AArch64::ADDSWri;
2670 case AArch64::ADDWrr:
2671 return AArch64::ADDSWrr;
2672 case AArch64::ADDWrs:
2673 return AArch64::ADDSWrs;
2674 case AArch64::ADDWrx:
2675 return AArch64::ADDSWrx;
2676 case AArch64::ANDWri:
2677 return AArch64::ANDSWri;
2678 case AArch64::ANDWrr:
2679 return AArch64::ANDSWrr;
2680 case AArch64::ANDWrs:
2681 return AArch64::ANDSWrs;
2682 case AArch64::BICWrr:
2683 return AArch64::BICSWrr;
2684 case AArch64::BICWrs:
2685 return AArch64::BICSWrs;
2686 case AArch64::SUBWri:
2687 return AArch64::SUBSWri;
2688 case AArch64::SUBWrr:
2689 return AArch64::SUBSWrr;
2690 case AArch64::SUBWrs:
2691 return AArch64::SUBSWrs;
2692 case AArch64::SUBWrx:
2693 return AArch64::SUBSWrx;
2695 case AArch64::ADDXri:
2696 return AArch64::ADDSXri;
2697 case AArch64::ADDXrr:
2698 return AArch64::ADDSXrr;
2699 case AArch64::ADDXrs:
2700 return AArch64::ADDSXrs;
2701 case AArch64::ADDXrx:
2702 return AArch64::ADDSXrx;
2703 case AArch64::ANDXri:
2704 return AArch64::ANDSXri;
2705 case AArch64::ANDXrr:
2706 return AArch64::ANDSXrr;
2707 case AArch64::ANDXrs:
2708 return AArch64::ANDSXrs;
2709 case AArch64::BICXrr:
2710 return AArch64::BICSXrr;
2711 case AArch64::BICXrs:
2712 return AArch64::BICSXrs;
2713 case AArch64::SUBXri:
2714 return AArch64::SUBSXri;
2715 case AArch64::SUBXrr:
2716 return AArch64::SUBSXrr;
2717 case AArch64::SUBXrs:
2718 return AArch64::SUBSXrs;
2719 case AArch64::SUBXrx:
2720 return AArch64::SUBSXrx;
2722 case AArch64::AND_PPzPP:
2723 return AArch64::ANDS_PPzPP;
2724 case AArch64::BIC_PPzPP:
2725 return AArch64::BICS_PPzPP;
2726 case AArch64::EOR_PPzPP:
2727 return AArch64::EORS_PPzPP;
2728 case AArch64::NAND_PPzPP:
2729 return AArch64::NANDS_PPzPP;
2730 case AArch64::NOR_PPzPP:
2731 return AArch64::NORS_PPzPP;
2732 case AArch64::ORN_PPzPP:
2733 return AArch64::ORNS_PPzPP;
2734 case AArch64::ORR_PPzPP:
2735 return AArch64::ORRS_PPzPP;
2736 case AArch64::BRKA_PPzP:
2737 return AArch64::BRKAS_PPzP;
2738 case AArch64::BRKPA_PPzPP:
2739 return AArch64::BRKPAS_PPzPP;
2740 case AArch64::BRKB_PPzP:
2741 return AArch64::BRKBS_PPzP;
2742 case AArch64::BRKPB_PPzPP:
2743 return AArch64::BRKPBS_PPzPP;
2744 case AArch64::BRKN_PPzP:
2745 return AArch64::BRKNS_PPzP;
2746 case AArch64::RDFFR_PPz:
2747 return AArch64::RDFFRS_PPz;
2748 case AArch64::PTRUE_B:
2749 return AArch64::PTRUES_B;
2760 if (
MI.hasOrderedMemoryRef())
2765 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2766 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2767 "Expected a reg or frame index operand.");
2771 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2773 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2786 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2787 Register BaseReg =
MI.getOperand(1).getReg();
2789 if (
MI.modifiesRegister(BaseReg,
TRI))
2802 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2804 MI.getMF()->getFunction().needsUnwindTableEntry();
2810 if (Subtarget.isPaired128Slow()) {
2811 switch (
MI.getOpcode()) {
2814 case AArch64::LDURQi:
2815 case AArch64::STURQi:
2816 case AArch64::LDRQui:
2817 case AArch64::STRQui:
2844std::optional<ExtAddrMode>
2849 bool OffsetIsScalable;
2850 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2851 return std::nullopt;
2854 return std::nullopt;
2869 int64_t OffsetScale = 1;
2874 case AArch64::LDURQi:
2875 case AArch64::STURQi:
2879 case AArch64::LDURDi:
2880 case AArch64::STURDi:
2881 case AArch64::LDURXi:
2882 case AArch64::STURXi:
2886 case AArch64::LDURWi:
2887 case AArch64::LDURSWi:
2888 case AArch64::STURWi:
2892 case AArch64::LDURHi:
2893 case AArch64::STURHi:
2894 case AArch64::LDURHHi:
2895 case AArch64::STURHHi:
2896 case AArch64::LDURSHXi:
2897 case AArch64::LDURSHWi:
2901 case AArch64::LDRBroX:
2902 case AArch64::LDRBBroX:
2903 case AArch64::LDRSBXroX:
2904 case AArch64::LDRSBWroX:
2905 case AArch64::STRBroX:
2906 case AArch64::STRBBroX:
2907 case AArch64::LDURBi:
2908 case AArch64::LDURBBi:
2909 case AArch64::LDURSBXi:
2910 case AArch64::LDURSBWi:
2911 case AArch64::STURBi:
2912 case AArch64::STURBBi:
2913 case AArch64::LDRBui:
2914 case AArch64::LDRBBui:
2915 case AArch64::LDRSBXui:
2916 case AArch64::LDRSBWui:
2917 case AArch64::STRBui:
2918 case AArch64::STRBBui:
2922 case AArch64::LDRQroX:
2923 case AArch64::STRQroX:
2924 case AArch64::LDRQui:
2925 case AArch64::STRQui:
2930 case AArch64::LDRDroX:
2931 case AArch64::STRDroX:
2932 case AArch64::LDRXroX:
2933 case AArch64::STRXroX:
2934 case AArch64::LDRDui:
2935 case AArch64::STRDui:
2936 case AArch64::LDRXui:
2937 case AArch64::STRXui:
2942 case AArch64::LDRWroX:
2943 case AArch64::LDRSWroX:
2944 case AArch64::STRWroX:
2945 case AArch64::LDRWui:
2946 case AArch64::LDRSWui:
2947 case AArch64::STRWui:
2952 case AArch64::LDRHroX:
2953 case AArch64::STRHroX:
2954 case AArch64::LDRHHroX:
2955 case AArch64::STRHHroX:
2956 case AArch64::LDRSHXroX:
2957 case AArch64::LDRSHWroX:
2958 case AArch64::LDRHui:
2959 case AArch64::STRHui:
2960 case AArch64::LDRHHui:
2961 case AArch64::STRHHui:
2962 case AArch64::LDRSHXui:
2963 case AArch64::LDRSHWui:
2971 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
2995 case AArch64::SBFMXri:
3008 AM.
Scale = OffsetScale;
3013 case TargetOpcode::SUBREG_TO_REG: {
3026 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3030 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3032 DefMI.getOperand(3).getImm() != 0)
3039 AM.
Scale = OffsetScale;
3050 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3051 int64_t NewOffset) ->
bool {
3052 int64_t MinOffset, MaxOffset;
3069 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3070 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3072 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3074 int64_t NewOffset = OldOffset + Disp;
3079 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3089 auto canFoldAddRegIntoAddrMode =
3106 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3107 Subtarget.isSTRQroSlow();
3116 case AArch64::ADDXri:
3122 return canFoldAddSubImmIntoAddrMode(Disp);
3124 case AArch64::SUBXri:
3130 return canFoldAddSubImmIntoAddrMode(-Disp);
3132 case AArch64::ADDXrs: {
3145 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3147 if (avoidSlowSTRQ(MemI))
3150 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3153 case AArch64::ADDXrr:
3161 if (!OptSize && avoidSlowSTRQ(MemI))
3163 return canFoldAddRegIntoAddrMode(1);
3165 case AArch64::ADDXrx:
3173 if (!OptSize && avoidSlowSTRQ(MemI))
3182 return canFoldAddRegIntoAddrMode(
3197 case AArch64::LDURQi:
3198 case AArch64::LDRQui:
3199 return AArch64::LDRQroX;
3200 case AArch64::STURQi:
3201 case AArch64::STRQui:
3202 return AArch64::STRQroX;
3203 case AArch64::LDURDi:
3204 case AArch64::LDRDui:
3205 return AArch64::LDRDroX;
3206 case AArch64::STURDi:
3207 case AArch64::STRDui:
3208 return AArch64::STRDroX;
3209 case AArch64::LDURXi:
3210 case AArch64::LDRXui:
3211 return AArch64::LDRXroX;
3212 case AArch64::STURXi:
3213 case AArch64::STRXui:
3214 return AArch64::STRXroX;
3215 case AArch64::LDURWi:
3216 case AArch64::LDRWui:
3217 return AArch64::LDRWroX;
3218 case AArch64::LDURSWi:
3219 case AArch64::LDRSWui:
3220 return AArch64::LDRSWroX;
3221 case AArch64::STURWi:
3222 case AArch64::STRWui:
3223 return AArch64::STRWroX;
3224 case AArch64::LDURHi:
3225 case AArch64::LDRHui:
3226 return AArch64::LDRHroX;
3227 case AArch64::STURHi:
3228 case AArch64::STRHui:
3229 return AArch64::STRHroX;
3230 case AArch64::LDURHHi:
3231 case AArch64::LDRHHui:
3232 return AArch64::LDRHHroX;
3233 case AArch64::STURHHi:
3234 case AArch64::STRHHui:
3235 return AArch64::STRHHroX;
3236 case AArch64::LDURSHXi:
3237 case AArch64::LDRSHXui:
3238 return AArch64::LDRSHXroX;
3239 case AArch64::LDURSHWi:
3240 case AArch64::LDRSHWui:
3241 return AArch64::LDRSHWroX;
3242 case AArch64::LDURBi:
3243 case AArch64::LDRBui:
3244 return AArch64::LDRBroX;
3245 case AArch64::LDURBBi:
3246 case AArch64::LDRBBui:
3247 return AArch64::LDRBBroX;
3248 case AArch64::LDURSBXi:
3249 case AArch64::LDRSBXui:
3250 return AArch64::LDRSBXroX;
3251 case AArch64::LDURSBWi:
3252 case AArch64::LDRSBWui:
3253 return AArch64::LDRSBWroX;
3254 case AArch64::STURBi:
3255 case AArch64::STRBui:
3256 return AArch64::STRBroX;
3257 case AArch64::STURBBi:
3258 case AArch64::STRBBui:
3259 return AArch64::STRBBroX;
3271 case AArch64::LDURQi:
3273 return AArch64::LDRQui;
3274 case AArch64::STURQi:
3276 return AArch64::STRQui;
3277 case AArch64::LDURDi:
3279 return AArch64::LDRDui;
3280 case AArch64::STURDi:
3282 return AArch64::STRDui;
3283 case AArch64::LDURXi:
3285 return AArch64::LDRXui;
3286 case AArch64::STURXi:
3288 return AArch64::STRXui;
3289 case AArch64::LDURWi:
3291 return AArch64::LDRWui;
3292 case AArch64::LDURSWi:
3294 return AArch64::LDRSWui;
3295 case AArch64::STURWi:
3297 return AArch64::STRWui;
3298 case AArch64::LDURHi:
3300 return AArch64::LDRHui;
3301 case AArch64::STURHi:
3303 return AArch64::STRHui;
3304 case AArch64::LDURHHi:
3306 return AArch64::LDRHHui;
3307 case AArch64::STURHHi:
3309 return AArch64::STRHHui;
3310 case AArch64::LDURSHXi:
3312 return AArch64::LDRSHXui;
3313 case AArch64::LDURSHWi:
3315 return AArch64::LDRSHWui;
3316 case AArch64::LDURBi:
3318 return AArch64::LDRBui;
3319 case AArch64::LDURBBi:
3321 return AArch64::LDRBBui;
3322 case AArch64::LDURSBXi:
3324 return AArch64::LDRSBXui;
3325 case AArch64::LDURSBWi:
3327 return AArch64::LDRSBWui;
3328 case AArch64::STURBi:
3330 return AArch64::STRBui;
3331 case AArch64::STURBBi:
3333 return AArch64::STRBBui;
3334 case AArch64::LDRQui:
3335 case AArch64::STRQui:
3338 case AArch64::LDRDui:
3339 case AArch64::STRDui:
3340 case AArch64::LDRXui:
3341 case AArch64::STRXui:
3344 case AArch64::LDRWui:
3345 case AArch64::LDRSWui:
3346 case AArch64::STRWui:
3349 case AArch64::LDRHui:
3350 case AArch64::STRHui:
3351 case AArch64::LDRHHui:
3352 case AArch64::STRHHui:
3353 case AArch64::LDRSHXui:
3354 case AArch64::LDRSHWui:
3357 case AArch64::LDRBui:
3358 case AArch64::LDRBBui:
3359 case AArch64::LDRSBXui:
3360 case AArch64::LDRSBWui:
3361 case AArch64::STRBui:
3362 case AArch64::STRBBui:
3376 case AArch64::LDURQi:
3377 case AArch64::STURQi:
3378 case AArch64::LDURDi:
3379 case AArch64::STURDi:
3380 case AArch64::LDURXi:
3381 case AArch64::STURXi:
3382 case AArch64::LDURWi:
3383 case AArch64::LDURSWi:
3384 case AArch64::STURWi:
3385 case AArch64::LDURHi:
3386 case AArch64::STURHi:
3387 case AArch64::LDURHHi:
3388 case AArch64::STURHHi:
3389 case AArch64::LDURSHXi:
3390 case AArch64::LDURSHWi:
3391 case AArch64::LDURBi:
3392 case AArch64::STURBi:
3393 case AArch64::LDURBBi:
3394 case AArch64::STURBBi:
3395 case AArch64::LDURSBWi:
3396 case AArch64::LDURSBXi:
3398 case AArch64::LDRQui:
3399 return AArch64::LDURQi;
3400 case AArch64::STRQui:
3401 return AArch64::STURQi;
3402 case AArch64::LDRDui:
3403 return AArch64::LDURDi;
3404 case AArch64::STRDui:
3405 return AArch64::STURDi;
3406 case AArch64::LDRXui:
3407 return AArch64::LDURXi;
3408 case AArch64::STRXui:
3409 return AArch64::STURXi;
3410 case AArch64::LDRWui:
3411 return AArch64::LDURWi;
3412 case AArch64::LDRSWui:
3413 return AArch64::LDURSWi;
3414 case AArch64::STRWui:
3415 return AArch64::STURWi;
3416 case AArch64::LDRHui:
3417 return AArch64::LDURHi;
3418 case AArch64::STRHui:
3419 return AArch64::STURHi;
3420 case AArch64::LDRHHui:
3421 return AArch64::LDURHHi;
3422 case AArch64::STRHHui:
3423 return AArch64::STURHHi;
3424 case AArch64::LDRSHXui:
3425 return AArch64::LDURSHXi;
3426 case AArch64::LDRSHWui:
3427 return AArch64::LDURSHWi;
3428 case AArch64::LDRBBui:
3429 return AArch64::LDURBBi;
3430 case AArch64::LDRBui:
3431 return AArch64::LDURBi;
3432 case AArch64::STRBBui:
3433 return AArch64::STURBBi;
3434 case AArch64::STRBui:
3435 return AArch64::STURBi;
3436 case AArch64::LDRSBWui:
3437 return AArch64::LDURSBWi;
3438 case AArch64::LDRSBXui:
3439 return AArch64::LDURSBXi;
3452 case AArch64::LDRQroX:
3453 case AArch64::LDURQi:
3454 case AArch64::LDRQui:
3455 return AArch64::LDRQroW;
3456 case AArch64::STRQroX:
3457 case AArch64::STURQi:
3458 case AArch64::STRQui:
3459 return AArch64::STRQroW;
3460 case AArch64::LDRDroX:
3461 case AArch64::LDURDi:
3462 case AArch64::LDRDui:
3463 return AArch64::LDRDroW;
3464 case AArch64::STRDroX:
3465 case AArch64::STURDi:
3466 case AArch64::STRDui:
3467 return AArch64::STRDroW;
3468 case AArch64::LDRXroX:
3469 case AArch64::LDURXi:
3470 case AArch64::LDRXui:
3471 return AArch64::LDRXroW;
3472 case AArch64::STRXroX:
3473 case AArch64::STURXi:
3474 case AArch64::STRXui:
3475 return AArch64::STRXroW;
3476 case AArch64::LDRWroX:
3477 case AArch64::LDURWi:
3478 case AArch64::LDRWui:
3479 return AArch64::LDRWroW;
3480 case AArch64::LDRSWroX:
3481 case AArch64::LDURSWi:
3482 case AArch64::LDRSWui:
3483 return AArch64::LDRSWroW;
3484 case AArch64::STRWroX:
3485 case AArch64::STURWi:
3486 case AArch64::STRWui:
3487 return AArch64::STRWroW;
3488 case AArch64::LDRHroX:
3489 case AArch64::LDURHi:
3490 case AArch64::LDRHui:
3491 return AArch64::LDRHroW;
3492 case AArch64::STRHroX:
3493 case AArch64::STURHi:
3494 case AArch64::STRHui:
3495 return AArch64::STRHroW;
3496 case AArch64::LDRHHroX:
3497 case AArch64::LDURHHi:
3498 case AArch64::LDRHHui:
3499 return AArch64::LDRHHroW;
3500 case AArch64::STRHHroX:
3501 case AArch64::STURHHi:
3502 case AArch64::STRHHui:
3503 return AArch64::STRHHroW;
3504 case AArch64::LDRSHXroX:
3505 case AArch64::LDURSHXi:
3506 case AArch64::LDRSHXui:
3507 return AArch64::LDRSHXroW;
3508 case AArch64::LDRSHWroX:
3509 case AArch64::LDURSHWi:
3510 case AArch64::LDRSHWui:
3511 return AArch64::LDRSHWroW;
3512 case AArch64::LDRBroX:
3513 case AArch64::LDURBi:
3514 case AArch64::LDRBui:
3515 return AArch64::LDRBroW;
3516 case AArch64::LDRBBroX:
3517 case AArch64::LDURBBi:
3518 case AArch64::LDRBBui:
3519 return AArch64::LDRBBroW;
3520 case AArch64::LDRSBXroX:
3521 case AArch64::LDURSBXi:
3522 case AArch64::LDRSBXui:
3523 return AArch64::LDRSBXroW;
3524 case AArch64::LDRSBWroX:
3525 case AArch64::LDURSBWi:
3526 case AArch64::LDRSBWui:
3527 return AArch64::LDRSBWroW;
3528 case AArch64::STRBroX:
3529 case AArch64::STURBi:
3530 case AArch64::STRBui:
3531 return AArch64::STRBroW;
3532 case AArch64::STRBBroX:
3533 case AArch64::STURBBi:
3534 case AArch64::STRBBui:
3535 return AArch64::STRBBroW;
3550 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3560 return B.getInstr();
3564 "Addressing mode not supported for folding");
3581 return B.getInstr();
3588 "Address offset can be a register or an immediate, but not both");
3590 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3595 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3609 return B.getInstr();
3613 "Function must not be called with an addressing mode it can't handle");
3622 case AArch64::LD1Fourv16b_POST:
3623 case AArch64::LD1Fourv1d_POST:
3624 case AArch64::LD1Fourv2d_POST:
3625 case AArch64::LD1Fourv2s_POST:
3626 case AArch64::LD1Fourv4h_POST:
3627 case AArch64::LD1Fourv4s_POST:
3628 case AArch64::LD1Fourv8b_POST:
3629 case AArch64::LD1Fourv8h_POST:
3630 case AArch64::LD1Onev16b_POST:
3631 case AArch64::LD1Onev1d_POST:
3632 case AArch64::LD1Onev2d_POST:
3633 case AArch64::LD1Onev2s_POST:
3634 case AArch64::LD1Onev4h_POST:
3635 case AArch64::LD1Onev4s_POST:
3636 case AArch64::LD1Onev8b_POST:
3637 case AArch64::LD1Onev8h_POST:
3638 case AArch64::LD1Rv16b_POST:
3639 case AArch64::LD1Rv1d_POST:
3640 case AArch64::LD1Rv2d_POST:
3641 case AArch64::LD1Rv2s_POST:
3642 case AArch64::LD1Rv4h_POST:
3643 case AArch64::LD1Rv4s_POST:
3644 case AArch64::LD1Rv8b_POST:
3645 case AArch64::LD1Rv8h_POST:
3646 case AArch64::LD1Threev16b_POST:
3647 case AArch64::LD1Threev1d_POST:
3648 case AArch64::LD1Threev2d_POST:
3649 case AArch64::LD1Threev2s_POST:
3650 case AArch64::LD1Threev4h_POST:
3651 case AArch64::LD1Threev4s_POST:
3652 case AArch64::LD1Threev8b_POST:
3653 case AArch64::LD1Threev8h_POST:
3654 case AArch64::LD1Twov16b_POST:
3655 case AArch64::LD1Twov1d_POST:
3656 case AArch64::LD1Twov2d_POST:
3657 case AArch64::LD1Twov2s_POST:
3658 case AArch64::LD1Twov4h_POST:
3659 case AArch64::LD1Twov4s_POST:
3660 case AArch64::LD1Twov8b_POST:
3661 case AArch64::LD1Twov8h_POST:
3662 case AArch64::LD1i16_POST:
3663 case AArch64::LD1i32_POST:
3664 case AArch64::LD1i64_POST:
3665 case AArch64::LD1i8_POST:
3666 case AArch64::LD2Rv16b_POST:
3667 case AArch64::LD2Rv1d_POST:
3668 case AArch64::LD2Rv2d_POST:
3669 case AArch64::LD2Rv2s_POST:
3670 case AArch64::LD2Rv4h_POST:
3671 case AArch64::LD2Rv4s_POST:
3672 case AArch64::LD2Rv8b_POST:
3673 case AArch64::LD2Rv8h_POST:
3674 case AArch64::LD2Twov16b_POST:
3675 case AArch64::LD2Twov2d_POST:
3676 case AArch64::LD2Twov2s_POST:
3677 case AArch64::LD2Twov4h_POST:
3678 case AArch64::LD2Twov4s_POST:
3679 case AArch64::LD2Twov8b_POST:
3680 case AArch64::LD2Twov8h_POST:
3681 case AArch64::LD2i16_POST:
3682 case AArch64::LD2i32_POST:
3683 case AArch64::LD2i64_POST:
3684 case AArch64::LD2i8_POST:
3685 case AArch64::LD3Rv16b_POST:
3686 case AArch64::LD3Rv1d_POST:
3687 case AArch64::LD3Rv2d_POST:
3688 case AArch64::LD3Rv2s_POST:
3689 case AArch64::LD3Rv4h_POST:
3690 case AArch64::LD3Rv4s_POST:
3691 case AArch64::LD3Rv8b_POST:
3692 case AArch64::LD3Rv8h_POST:
3693 case AArch64::LD3Threev16b_POST:
3694 case AArch64::LD3Threev2d_POST:
3695 case AArch64::LD3Threev2s_POST:
3696 case AArch64::LD3Threev4h_POST:
3697 case AArch64::LD3Threev4s_POST:
3698 case AArch64::LD3Threev8b_POST:
3699 case AArch64::LD3Threev8h_POST:
3700 case AArch64::LD3i16_POST:
3701 case AArch64::LD3i32_POST:
3702 case AArch64::LD3i64_POST:
3703 case AArch64::LD3i8_POST:
3704 case AArch64::LD4Fourv16b_POST:
3705 case AArch64::LD4Fourv2d_POST:
3706 case AArch64::LD4Fourv2s_POST:
3707 case AArch64::LD4Fourv4h_POST:
3708 case AArch64::LD4Fourv4s_POST:
3709 case AArch64::LD4Fourv8b_POST:
3710 case AArch64::LD4Fourv8h_POST:
3711 case AArch64::LD4Rv16b_POST:
3712 case AArch64::LD4Rv1d_POST:
3713 case AArch64::LD4Rv2d_POST:
3714 case AArch64::LD4Rv2s_POST:
3715 case AArch64::LD4Rv4h_POST:
3716 case AArch64::LD4Rv4s_POST:
3717 case AArch64::LD4Rv8b_POST:
3718 case AArch64::LD4Rv8h_POST:
3719 case AArch64::LD4i16_POST:
3720 case AArch64::LD4i32_POST:
3721 case AArch64::LD4i64_POST:
3722 case AArch64::LD4i8_POST:
3723 case AArch64::LDAPRWpost:
3724 case AArch64::LDAPRXpost:
3725 case AArch64::LDIAPPWpost:
3726 case AArch64::LDIAPPXpost:
3727 case AArch64::LDPDpost:
3728 case AArch64::LDPQpost:
3729 case AArch64::LDPSWpost:
3730 case AArch64::LDPSpost:
3731 case AArch64::LDPWpost:
3732 case AArch64::LDPXpost:
3733 case AArch64::LDRBBpost:
3734 case AArch64::LDRBpost:
3735 case AArch64::LDRDpost:
3736 case AArch64::LDRHHpost:
3737 case AArch64::LDRHpost:
3738 case AArch64::LDRQpost:
3739 case AArch64::LDRSBWpost:
3740 case AArch64::LDRSBXpost:
3741 case AArch64::LDRSHWpost:
3742 case AArch64::LDRSHXpost:
3743 case AArch64::LDRSWpost:
3744 case AArch64::LDRSpost:
3745 case AArch64::LDRWpost:
3746 case AArch64::LDRXpost:
3747 case AArch64::ST1Fourv16b_POST:
3748 case AArch64::ST1Fourv1d_POST:
3749 case AArch64::ST1Fourv2d_POST:
3750 case AArch64::ST1Fourv2s_POST:
3751 case AArch64::ST1Fourv4h_POST:
3752 case AArch64::ST1Fourv4s_POST:
3753 case AArch64::ST1Fourv8b_POST:
3754 case AArch64::ST1Fourv8h_POST:
3755 case AArch64::ST1Onev16b_POST:
3756 case AArch64::ST1Onev1d_POST:
3757 case AArch64::ST1Onev2d_POST:
3758 case AArch64::ST1Onev2s_POST:
3759 case AArch64::ST1Onev4h_POST:
3760 case AArch64::ST1Onev4s_POST:
3761 case AArch64::ST1Onev8b_POST:
3762 case AArch64::ST1Onev8h_POST:
3763 case AArch64::ST1Threev16b_POST:
3764 case AArch64::ST1Threev1d_POST:
3765 case AArch64::ST1Threev2d_POST:
3766 case AArch64::ST1Threev2s_POST:
3767 case AArch64::ST1Threev4h_POST:
3768 case AArch64::ST1Threev4s_POST:
3769 case AArch64::ST1Threev8b_POST:
3770 case AArch64::ST1Threev8h_POST:
3771 case AArch64::ST1Twov16b_POST:
3772 case AArch64::ST1Twov1d_POST:
3773 case AArch64::ST1Twov2d_POST:
3774 case AArch64::ST1Twov2s_POST:
3775 case AArch64::ST1Twov4h_POST:
3776 case AArch64::ST1Twov4s_POST:
3777 case AArch64::ST1Twov8b_POST:
3778 case AArch64::ST1Twov8h_POST:
3779 case AArch64::ST1i16_POST:
3780 case AArch64::ST1i32_POST:
3781 case AArch64::ST1i64_POST:
3782 case AArch64::ST1i8_POST:
3783 case AArch64::ST2GPostIndex:
3784 case AArch64::ST2Twov16b_POST:
3785 case AArch64::ST2Twov2d_POST:
3786 case AArch64::ST2Twov2s_POST:
3787 case AArch64::ST2Twov4h_POST:
3788 case AArch64::ST2Twov4s_POST:
3789 case AArch64::ST2Twov8b_POST:
3790 case AArch64::ST2Twov8h_POST:
3791 case AArch64::ST2i16_POST:
3792 case AArch64::ST2i32_POST:
3793 case AArch64::ST2i64_POST:
3794 case AArch64::ST2i8_POST:
3795 case AArch64::ST3Threev16b_POST:
3796 case AArch64::ST3Threev2d_POST:
3797 case AArch64::ST3Threev2s_POST:
3798 case AArch64::ST3Threev4h_POST:
3799 case AArch64::ST3Threev4s_POST:
3800 case AArch64::ST3Threev8b_POST:
3801 case AArch64::ST3Threev8h_POST:
3802 case AArch64::ST3i16_POST:
3803 case AArch64::ST3i32_POST:
3804 case AArch64::ST3i64_POST:
3805 case AArch64::ST3i8_POST:
3806 case AArch64::ST4Fourv16b_POST:
3807 case AArch64::ST4Fourv2d_POST:
3808 case AArch64::ST4Fourv2s_POST:
3809 case AArch64::ST4Fourv4h_POST:
3810 case AArch64::ST4Fourv4s_POST:
3811 case AArch64::ST4Fourv8b_POST:
3812 case AArch64::ST4Fourv8h_POST:
3813 case AArch64::ST4i16_POST:
3814 case AArch64::ST4i32_POST:
3815 case AArch64::ST4i64_POST:
3816 case AArch64::ST4i8_POST:
3817 case AArch64::STGPostIndex:
3818 case AArch64::STGPpost:
3819 case AArch64::STPDpost:
3820 case AArch64::STPQpost:
3821 case AArch64::STPSpost:
3822 case AArch64::STPWpost:
3823 case AArch64::STPXpost:
3824 case AArch64::STRBBpost:
3825 case AArch64::STRBpost:
3826 case AArch64::STRDpost:
3827 case AArch64::STRHHpost:
3828 case AArch64::STRHpost:
3829 case AArch64::STRQpost:
3830 case AArch64::STRSpost:
3831 case AArch64::STRWpost:
3832 case AArch64::STRXpost:
3833 case AArch64::STZ2GPostIndex:
3834 case AArch64::STZGPostIndex:
3841 bool &OffsetIsScalable,
TypeSize &Width,
3862 int64_t Dummy1, Dummy2;
3884 return BaseOp->
isReg() || BaseOp->
isFI();
3891 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3896 TypeSize &Width, int64_t &MinOffset,
3897 int64_t &MaxOffset) {
3903 MinOffset = MaxOffset = 0;
3906 case AArch64::LDRQui:
3907 case AArch64::STRQui:
3913 case AArch64::LDRXui:
3914 case AArch64::LDRDui:
3915 case AArch64::STRXui:
3916 case AArch64::STRDui:
3917 case AArch64::PRFMui:
3923 case AArch64::LDRWui:
3924 case AArch64::LDRSui:
3925 case AArch64::LDRSWui:
3926 case AArch64::STRWui:
3927 case AArch64::STRSui:
3933 case AArch64::LDRHui:
3934 case AArch64::LDRHHui:
3935 case AArch64::LDRSHWui:
3936 case AArch64::LDRSHXui:
3937 case AArch64::STRHui:
3938 case AArch64::STRHHui:
3944 case AArch64::LDRBui:
3945 case AArch64::LDRBBui:
3946 case AArch64::LDRSBWui:
3947 case AArch64::LDRSBXui:
3948 case AArch64::STRBui:
3949 case AArch64::STRBBui:
3956 case AArch64::STRQpre:
3957 case AArch64::LDRQpost:
3963 case AArch64::LDRDpost:
3964 case AArch64::LDRDpre:
3965 case AArch64::LDRXpost:
3966 case AArch64::LDRXpre:
3967 case AArch64::STRDpost:
3968 case AArch64::STRDpre:
3969 case AArch64::STRXpost:
3970 case AArch64::STRXpre:
3976 case AArch64::STRWpost:
3977 case AArch64::STRWpre:
3978 case AArch64::LDRWpost:
3979 case AArch64::LDRWpre:
3980 case AArch64::STRSpost:
3981 case AArch64::STRSpre:
3982 case AArch64::LDRSpost:
3983 case AArch64::LDRSpre:
3989 case AArch64::LDRHpost:
3990 case AArch64::LDRHpre:
3991 case AArch64::STRHpost:
3992 case AArch64::STRHpre:
3993 case AArch64::LDRHHpost:
3994 case AArch64::LDRHHpre:
3995 case AArch64::STRHHpost:
3996 case AArch64::STRHHpre:
4002 case AArch64::LDRBpost:
4003 case AArch64::LDRBpre:
4004 case AArch64::STRBpost:
4005 case AArch64::STRBpre:
4006 case AArch64::LDRBBpost:
4007 case AArch64::LDRBBpre:
4008 case AArch64::STRBBpost:
4009 case AArch64::STRBBpre:
4016 case AArch64::LDURQi:
4017 case AArch64::STURQi:
4023 case AArch64::LDURXi:
4024 case AArch64::LDURDi:
4025 case AArch64::LDAPURXi:
4026 case AArch64::STURXi:
4027 case AArch64::STURDi:
4028 case AArch64::STLURXi:
4029 case AArch64::PRFUMi:
4035 case AArch64::LDURWi:
4036 case AArch64::LDURSi:
4037 case AArch64::LDURSWi:
4038 case AArch64::LDAPURi:
4039 case AArch64::LDAPURSWi:
4040 case AArch64::STURWi:
4041 case AArch64::STURSi:
4042 case AArch64::STLURWi:
4048 case AArch64::LDURHi:
4049 case AArch64::LDURHHi:
4050 case AArch64::LDURSHXi:
4051 case AArch64::LDURSHWi:
4052 case AArch64::LDAPURHi:
4053 case AArch64::LDAPURSHWi:
4054 case AArch64::LDAPURSHXi:
4055 case AArch64::STURHi:
4056 case AArch64::STURHHi:
4057 case AArch64::STLURHi:
4063 case AArch64::LDURBi:
4064 case AArch64::LDURBBi:
4065 case AArch64::LDURSBXi:
4066 case AArch64::LDURSBWi:
4067 case AArch64::LDAPURBi:
4068 case AArch64::LDAPURSBWi:
4069 case AArch64::LDAPURSBXi:
4070 case AArch64::STURBi:
4071 case AArch64::STURBBi:
4072 case AArch64::STLURBi:
4079 case AArch64::LDPQi:
4080 case AArch64::LDNPQi:
4081 case AArch64::STPQi:
4082 case AArch64::STNPQi:
4083 case AArch64::LDPQpost:
4084 case AArch64::LDPQpre:
4085 case AArch64::STPQpost:
4086 case AArch64::STPQpre:
4092 case AArch64::LDPXi:
4093 case AArch64::LDPDi:
4094 case AArch64::LDNPXi:
4095 case AArch64::LDNPDi:
4096 case AArch64::STPXi:
4097 case AArch64::STPDi:
4098 case AArch64::STNPXi:
4099 case AArch64::STNPDi:
4100 case AArch64::LDPDpost:
4101 case AArch64::LDPDpre:
4102 case AArch64::LDPXpost:
4103 case AArch64::LDPXpre:
4104 case AArch64::STPDpost:
4105 case AArch64::STPDpre:
4106 case AArch64::STPXpost:
4107 case AArch64::STPXpre:
4113 case AArch64::LDPWi:
4114 case AArch64::LDPSi:
4115 case AArch64::LDNPWi:
4116 case AArch64::LDNPSi:
4117 case AArch64::STPWi:
4118 case AArch64::STPSi:
4119 case AArch64::STNPWi:
4120 case AArch64::STNPSi:
4121 case AArch64::LDPSpost:
4122 case AArch64::LDPSpre:
4123 case AArch64::LDPWpost:
4124 case AArch64::LDPWpre:
4125 case AArch64::STPSpost:
4126 case AArch64::STPSpre:
4127 case AArch64::STPWpost:
4128 case AArch64::STPWpre:
4134 case AArch64::StoreSwiftAsyncContext:
4147 case AArch64::TAGPstack:
4157 case AArch64::STZGi:
4164 case AArch64::STR_ZZZZXI:
4165 case AArch64::LDR_ZZZZXI:
4171 case AArch64::STR_ZZZXI:
4172 case AArch64::LDR_ZZZXI:
4178 case AArch64::STR_ZZXI:
4179 case AArch64::LDR_ZZXI:
4185 case AArch64::LDR_PXI:
4186 case AArch64::STR_PXI:
4192 case AArch64::LDR_PPXI:
4193 case AArch64::STR_PPXI:
4199 case AArch64::LDR_ZXI:
4200 case AArch64::STR_ZXI:
4206 case AArch64::LD1B_IMM:
4207 case AArch64::LD1H_IMM:
4208 case AArch64::LD1W_IMM:
4209 case AArch64::LD1D_IMM:
4210 case AArch64::LDNT1B_ZRI:
4211 case AArch64::LDNT1H_ZRI:
4212 case AArch64::LDNT1W_ZRI:
4213 case AArch64::LDNT1D_ZRI:
4214 case AArch64::ST1B_IMM:
4215 case AArch64::ST1H_IMM:
4216 case AArch64::ST1W_IMM:
4217 case AArch64::ST1D_IMM:
4218 case AArch64::STNT1B_ZRI:
4219 case AArch64::STNT1H_ZRI:
4220 case AArch64::STNT1W_ZRI:
4221 case AArch64::STNT1D_ZRI:
4222 case AArch64::LDNF1B_IMM:
4223 case AArch64::LDNF1H_IMM:
4224 case AArch64::LDNF1W_IMM:
4225 case AArch64::LDNF1D_IMM:
4233 case AArch64::LD2B_IMM:
4234 case AArch64::LD2H_IMM:
4235 case AArch64::LD2W_IMM:
4236 case AArch64::LD2D_IMM:
4237 case AArch64::ST2B_IMM:
4238 case AArch64::ST2H_IMM:
4239 case AArch64::ST2W_IMM:
4240 case AArch64::ST2D_IMM:
4246 case AArch64::LD3B_IMM:
4247 case AArch64::LD3H_IMM:
4248 case AArch64::LD3W_IMM:
4249 case AArch64::LD3D_IMM:
4250 case AArch64::ST3B_IMM:
4251 case AArch64::ST3H_IMM:
4252 case AArch64::ST3W_IMM:
4253 case AArch64::ST3D_IMM:
4259 case AArch64::LD4B_IMM:
4260 case AArch64::LD4H_IMM:
4261 case AArch64::LD4W_IMM:
4262 case AArch64::LD4D_IMM:
4263 case AArch64::ST4B_IMM:
4264 case AArch64::ST4H_IMM:
4265 case AArch64::ST4W_IMM:
4266 case AArch64::ST4D_IMM:
4272 case AArch64::LD1B_H_IMM:
4273 case AArch64::LD1SB_H_IMM:
4274 case AArch64::LD1H_S_IMM:
4275 case AArch64::LD1SH_S_IMM:
4276 case AArch64::LD1W_D_IMM:
4277 case AArch64::LD1SW_D_IMM:
4278 case AArch64::ST1B_H_IMM:
4279 case AArch64::ST1H_S_IMM:
4280 case AArch64::ST1W_D_IMM:
4281 case AArch64::LDNF1B_H_IMM:
4282 case AArch64::LDNF1SB_H_IMM:
4283 case AArch64::LDNF1H_S_IMM:
4284 case AArch64::LDNF1SH_S_IMM:
4285 case AArch64::LDNF1W_D_IMM:
4286 case AArch64::LDNF1SW_D_IMM:
4294 case AArch64::LD1B_S_IMM:
4295 case AArch64::LD1SB_S_IMM:
4296 case AArch64::LD1H_D_IMM:
4297 case AArch64::LD1SH_D_IMM:
4298 case AArch64::ST1B_S_IMM:
4299 case AArch64::ST1H_D_IMM:
4300 case AArch64::LDNF1B_S_IMM:
4301 case AArch64::LDNF1SB_S_IMM:
4302 case AArch64::LDNF1H_D_IMM:
4303 case AArch64::LDNF1SH_D_IMM:
4311 case AArch64::LD1B_D_IMM:
4312 case AArch64::LD1SB_D_IMM:
4313 case AArch64::ST1B_D_IMM:
4314 case AArch64::LDNF1B_D_IMM:
4315 case AArch64::LDNF1SB_D_IMM:
4323 case AArch64::ST2Gi:
4324 case AArch64::STZ2Gi:
4330 case AArch64::STGPi:
4336 case AArch64::LD1RB_IMM:
4337 case AArch64::LD1RB_H_IMM:
4338 case AArch64::LD1RB_S_IMM:
4339 case AArch64::LD1RB_D_IMM:
4340 case AArch64::LD1RSB_H_IMM:
4341 case AArch64::LD1RSB_S_IMM:
4342 case AArch64::LD1RSB_D_IMM:
4348 case AArch64::LD1RH_IMM:
4349 case AArch64::LD1RH_S_IMM:
4350 case AArch64::LD1RH_D_IMM:
4351 case AArch64::LD1RSH_S_IMM:
4352 case AArch64::LD1RSH_D_IMM:
4358 case AArch64::LD1RW_IMM:
4359 case AArch64::LD1RW_D_IMM:
4360 case AArch64::LD1RSW_IMM:
4366 case AArch64::LD1RD_IMM:
4382 case AArch64::LDRBBui:
4383 case AArch64::LDURBBi:
4384 case AArch64::LDRSBWui:
4385 case AArch64::LDURSBWi:
4386 case AArch64::STRBBui:
4387 case AArch64::STURBBi:
4389 case AArch64::LDRHHui:
4390 case AArch64::LDURHHi:
4391 case AArch64::LDRSHWui:
4392 case AArch64::LDURSHWi:
4393 case AArch64::STRHHui:
4394 case AArch64::STURHHi:
4396 case AArch64::LDRSui:
4397 case AArch64::LDURSi:
4398 case AArch64::LDRSpre:
4399 case AArch64::LDRSWui:
4400 case AArch64::LDURSWi:
4401 case AArch64::LDRSWpre:
4402 case AArch64::LDRWpre:
4403 case AArch64::LDRWui:
4404 case AArch64::LDURWi:
4405 case AArch64::STRSui:
4406 case AArch64::STURSi:
4407 case AArch64::STRSpre:
4408 case AArch64::STRWui:
4409 case AArch64::STURWi:
4410 case AArch64::STRWpre:
4411 case AArch64::LDPSi:
4412 case AArch64::LDPSWi:
4413 case AArch64::LDPWi:
4414 case AArch64::STPSi:
4415 case AArch64::STPWi:
4417 case AArch64::LDRDui:
4418 case AArch64::LDURDi:
4419 case AArch64::LDRDpre:
4420 case AArch64::LDRXui:
4421 case AArch64::LDURXi:
4422 case AArch64::LDRXpre:
4423 case AArch64::STRDui:
4424 case AArch64::STURDi:
4425 case AArch64::STRDpre:
4426 case AArch64::STRXui:
4427 case AArch64::STURXi:
4428 case AArch64::STRXpre:
4429 case AArch64::LDPDi:
4430 case AArch64::LDPXi:
4431 case AArch64::STPDi:
4432 case AArch64::STPXi:
4434 case AArch64::LDRQui:
4435 case AArch64::LDURQi:
4436 case AArch64::STRQui:
4437 case AArch64::STURQi:
4438 case AArch64::STRQpre:
4439 case AArch64::LDPQi:
4440 case AArch64::LDRQpre:
4441 case AArch64::STPQi:
4443 case AArch64::STZGi:
4444 case AArch64::ST2Gi:
4445 case AArch64::STZ2Gi:
4446 case AArch64::STGPi:
4452 switch (
MI.getOpcode()) {
4455 case AArch64::LDRWpre:
4456 case AArch64::LDRXpre:
4457 case AArch64::LDRSWpre:
4458 case AArch64::LDRSpre:
4459 case AArch64::LDRDpre:
4460 case AArch64::LDRQpre:
4466 switch (
MI.getOpcode()) {
4469 case AArch64::STRWpre:
4470 case AArch64::STRXpre:
4471 case AArch64::STRSpre:
4472 case AArch64::STRDpre:
4473 case AArch64::STRQpre:
4483 switch (
MI.getOpcode()) {
4486 case AArch64::LDPSi:
4487 case AArch64::LDPSWi:
4488 case AArch64::LDPDi:
4489 case AArch64::LDPQi:
4490 case AArch64::LDPWi:
4491 case AArch64::LDPXi:
4492 case AArch64::STPSi:
4493 case AArch64::STPDi:
4494 case AArch64::STPQi:
4495 case AArch64::STPWi:
4496 case AArch64::STPXi:
4497 case AArch64::STGPi:
4503 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4507 return MI.getOperand(
Idx);
4512 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4516 return MI.getOperand(
Idx);
4521 switch (
MI.getOpcode()) {
4524 case AArch64::LDRBroX:
4525 case AArch64::LDRBBroX:
4526 case AArch64::LDRSBXroX:
4527 case AArch64::LDRSBWroX:
4528 case AArch64::LDRHroX:
4529 case AArch64::LDRHHroX:
4530 case AArch64::LDRSHXroX:
4531 case AArch64::LDRSHWroX:
4532 case AArch64::LDRWroX:
4533 case AArch64::LDRSroX:
4534 case AArch64::LDRSWroX:
4535 case AArch64::LDRDroX:
4536 case AArch64::LDRXroX:
4537 case AArch64::LDRQroX:
4538 return MI.getOperand(4);
4544 if (
MI.getParent() ==
nullptr)
4554 auto Reg =
Op.getReg();
4555 if (Reg.isPhysical())
4556 return AArch64::FPR16RegClass.
contains(Reg);
4558 return TRC == &AArch64::FPR16RegClass ||
4559 TRC == &AArch64::FPR16_loRegClass;
4568 auto Reg =
Op.getReg();
4569 if (Reg.isPhysical())
4570 return AArch64::FPR128RegClass.
contains(Reg);
4572 return TRC == &AArch64::FPR128RegClass ||
4573 TRC == &AArch64::FPR128_loRegClass;
4579 switch (
MI.getOpcode()) {
4582 case AArch64::PACIASP:
4583 case AArch64::PACIBSP:
4586 case AArch64::PAUTH_PROLOGUE:
4589 case AArch64::HINT: {
4590 unsigned Imm =
MI.getOperand(0).getImm();
4592 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4595 if (Imm == 25 || Imm == 27)
4607 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4608 return AArch64::FPR128RegClass.contains(Reg) ||
4609 AArch64::FPR64RegClass.contains(Reg) ||
4610 AArch64::FPR32RegClass.contains(Reg) ||
4611 AArch64::FPR16RegClass.contains(Reg) ||
4612 AArch64::FPR8RegClass.contains(Reg);
4619 auto Reg =
Op.getReg();
4620 if (Reg.isPhysical())
4624 return TRC == &AArch64::FPR128RegClass ||
4625 TRC == &AArch64::FPR128_loRegClass ||
4626 TRC == &AArch64::FPR64RegClass ||
4627 TRC == &AArch64::FPR64_loRegClass ||
4628 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4629 TRC == &AArch64::FPR8RegClass;
4651 if (FirstOpc == SecondOpc)
4657 case AArch64::STRSui:
4658 case AArch64::STURSi:
4659 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4660 case AArch64::STRDui:
4661 case AArch64::STURDi:
4662 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4663 case AArch64::STRQui:
4664 case AArch64::STURQi:
4665 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4666 case AArch64::STRWui:
4667 case AArch64::STURWi:
4668 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4669 case AArch64::STRXui:
4670 case AArch64::STURXi:
4671 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4672 case AArch64::LDRSui:
4673 case AArch64::LDURSi:
4674 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4675 case AArch64::LDRDui:
4676 case AArch64::LDURDi:
4677 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4678 case AArch64::LDRQui:
4679 case AArch64::LDURQi:
4680 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4681 case AArch64::LDRWui:
4682 case AArch64::LDURWi:
4683 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4684 case AArch64::LDRSWui:
4685 case AArch64::LDURSWi:
4686 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4687 case AArch64::LDRXui:
4688 case AArch64::LDURXi:
4689 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4696 int64_t Offset1,
unsigned Opcode1,
int FI2,
4697 int64_t Offset2,
unsigned Opcode2) {
4703 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4706 if (ObjectOffset1 % Scale1 != 0)
4708 ObjectOffset1 /= Scale1;
4710 if (ObjectOffset2 % Scale2 != 0)
4712 ObjectOffset2 /= Scale2;
4713 ObjectOffset1 += Offset1;
4714 ObjectOffset2 += Offset2;
4715 return ObjectOffset1 + 1 == ObjectOffset2;
4727 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4728 unsigned NumBytes)
const {
4738 "Only base registers and frame indices are supported.");
4745 if (ClusterSize > 2)
4752 unsigned FirstOpc = FirstLdSt.
getOpcode();
4753 unsigned SecondOpc = SecondLdSt.
getOpcode();
4773 if (Offset1 > 63 || Offset1 < -64)
4778 if (BaseOp1.
isFI()) {
4780 "Caller should have ordered offsets.");
4785 BaseOp2.
getIndex(), Offset2, SecondOpc);
4788 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4790 return Offset1 + 1 == Offset2;
4794 unsigned Reg,
unsigned SubIdx,
4798 return MIB.
addReg(Reg, State);
4801 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4802 return MIB.
addReg(Reg, State, SubIdx);
4809 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4818 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4820 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4821 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4822 unsigned NumRegs = Indices.
size();
4824 int SubReg = 0,
End = NumRegs, Incr = 1;
4842 unsigned SrcReg,
bool KillSrc,
4843 unsigned Opcode,
unsigned ZeroReg,
4846 unsigned NumRegs = Indices.
size();
4849 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4850 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4851 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4852 "GPR reg sequences should not be able to overlap");
4868 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4869 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4872 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4874 if (Subtarget.hasZeroCycleRegMove()) {
4877 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4879 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4895 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4900 if (Subtarget.hasZeroCycleRegMove()) {
4903 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4905 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4925 if (AArch64::PPRRegClass.
contains(DestReg) &&
4926 AArch64::PPRRegClass.
contains(SrcReg)) {
4928 "Unexpected SVE register.");
4938 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4939 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4940 if (DestIsPNR || SrcIsPNR) {
4942 return (R - AArch64::PN0) + AArch64::P0;
4944 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4945 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4947 if (PPRSrcReg != PPRDestReg) {
4959 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4960 AArch64::ZPRRegClass.
contains(SrcReg)) {
4962 "Unexpected SVE register.");
4970 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
4971 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
4972 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
4973 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
4975 "Unexpected SVE register.");
4976 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
4983 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
4984 AArch64::ZPR3RegClass.
contains(SrcReg)) {
4986 "Unexpected SVE register.");
4987 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4995 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
4996 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
4997 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
4998 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5000 "Unexpected SVE register.");
5001 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5002 AArch64::zsub2, AArch64::zsub3};
5008 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5009 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
5010 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5016 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
5030 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5031 AArch64::DDDDRegClass.
contains(SrcReg)) {
5032 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5033 AArch64::dsub2, AArch64::dsub3};
5040 if (AArch64::DDDRegClass.
contains(DestReg) &&
5041 AArch64::DDDRegClass.
contains(SrcReg)) {
5042 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5050 if (AArch64::DDRegClass.
contains(DestReg) &&
5051 AArch64::DDRegClass.
contains(SrcReg)) {
5052 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5059 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5060 AArch64::QQQQRegClass.
contains(SrcReg)) {
5061 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5062 AArch64::qsub2, AArch64::qsub3};
5069 if (AArch64::QQQRegClass.
contains(DestReg) &&
5070 AArch64::QQQRegClass.
contains(SrcReg)) {
5071 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5079 if (AArch64::QQRegClass.
contains(DestReg) &&
5080 AArch64::QQRegClass.
contains(SrcReg)) {
5081 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5087 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5088 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5089 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5091 AArch64::XZR, Indices);
5095 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5096 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5097 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5099 AArch64::WZR, Indices);
5103 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5104 AArch64::FPR128RegClass.
contains(SrcReg)) {
5109 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5110 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5130 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5131 AArch64::FPR64RegClass.
contains(SrcReg)) {
5137 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5138 AArch64::FPR32RegClass.
contains(SrcReg)) {
5144 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5145 AArch64::FPR16RegClass.
contains(SrcReg)) {
5147 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
5149 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
5155 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5156 AArch64::FPR8RegClass.
contains(SrcReg)) {
5158 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
5160 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
5167 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5168 AArch64::GPR64RegClass.
contains(SrcReg)) {
5173 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5174 AArch64::FPR64RegClass.
contains(SrcReg)) {
5180 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5181 AArch64::GPR32RegClass.
contains(SrcReg)) {
5186 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5187 AArch64::FPR32RegClass.
contains(SrcReg)) {
5193 if (DestReg == AArch64::NZCV) {
5194 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5196 .
addImm(AArch64SysReg::NZCV)
5202 if (SrcReg == AArch64::NZCV) {
5203 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
5205 .
addImm(AArch64SysReg::NZCV)
5212 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
5213 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
5223 unsigned SubIdx0,
unsigned SubIdx1,
int FI,