59#define GET_INSTRINFO_CTOR_DTOR
60#include "AArch64GenInstrInfo.inc"
64 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
68 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
72 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
76 cl::desc(
"Restrict range of B instructions (DEBUG)"));
81 RI(STI.getTargetTriple()), Subtarget(STI) {}
92 auto Op =
MI.getOpcode();
93 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
94 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
98 if (
MI.isMetaInstruction())
103 unsigned NumBytes = 0;
109 switch (
Desc.getOpcode()) {
112 return Desc.getSize();
119 case TargetOpcode::STACKMAP:
122 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
124 case TargetOpcode::PATCHPOINT:
127 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
129 case TargetOpcode::STATEPOINT:
131 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
136 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
141 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
143 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
144 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
148 case TargetOpcode::PATCHABLE_EVENT_CALL:
154 NumBytes =
MI.getOperand(1).getImm();
156 case TargetOpcode::BUNDLE:
157 NumBytes = getInstBundleLength(
MI);
164unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
168 while (++
I !=
E &&
I->isInsideBundle()) {
169 assert(!
I->isBundle() &&
"No nested bundle!");
228 int64_t BrOffset)
const {
230 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
231 "over conditional branch expansion");
232 return isIntN(Bits, BrOffset / 4);
237 switch (
MI.getOpcode()) {
241 return MI.getOperand(0).getMBB();
246 return MI.getOperand(2).getMBB();
252 return MI.getOperand(1).getMBB();
262 assert(RS &&
"RegScavenger required for long branching");
264 "new block should be inserted for expanding unconditional branch");
267 "restore block should be inserted for restoring clobbered registers");
272 if (!isInt<33>(BrOffset))
274 "Branch offsets outside of the signed 33-bit range not supported");
288 constexpr Register Reg = AArch64::X16;
290 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
298 if (Scavenged != AArch64::NoRegister &&
300 buildIndirectBranch(Scavenged, NewDestBB);
310 "Unable to insert indirect branch inside function that has red zone");
333 bool AllowModify)
const {
340 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
341 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
345 if (!isUnpredicatedTerminator(*
I))
352 unsigned LastOpc = LastInst->
getOpcode();
353 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
368 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
375 LastInst = SecondLastInst;
377 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
382 SecondLastInst = &*
I;
383 SecondLastOpc = SecondLastInst->
getOpcode();
394 LastInst = SecondLastInst;
396 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
398 "unreachable unconditional branches removed above");
407 SecondLastInst = &*
I;
408 SecondLastOpc = SecondLastInst->
getOpcode();
412 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
428 I->eraseFromParent();
437 I->eraseFromParent();
446 MachineBranchPredicate &MBP,
447 bool AllowModify)
const {
457 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
458 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
462 if (!isUnpredicatedTerminator(*
I))
467 unsigned LastOpc = LastInst->
getOpcode();
482 assert(MBP.TrueDest &&
"expected!");
485 MBP.ConditionDef =
nullptr;
486 MBP.SingleUseCondition =
false;
490 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
491 : MachineBranchPredicate::PRED_EQ;
497 if (
Cond[0].getImm() != -1) {
503 switch (
Cond[1].getImm()) {
507 Cond[1].setImm(AArch64::CBNZW);
510 Cond[1].setImm(AArch64::CBZW);
513 Cond[1].setImm(AArch64::CBNZX);
516 Cond[1].setImm(AArch64::CBZX);
519 Cond[1].setImm(AArch64::TBNZW);
522 Cond[1].setImm(AArch64::TBZW);
525 Cond[1].setImm(AArch64::TBNZX);
528 Cond[1].setImm(AArch64::TBZX);
537 int *BytesRemoved)
const {
547 I->eraseFromParent();
564 I->eraseFromParent();
571void AArch64InstrInfo::instantiateCondBranch(
574 if (
Cond[0].getImm() != -1) {
592 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
631 unsigned *NewVReg =
nullptr) {
636 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
639 unsigned SrcOpNum = 0;
641 case AArch64::ADDSXri:
642 case AArch64::ADDSWri:
648 case AArch64::ADDXri:
649 case AArch64::ADDWri:
655 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
658 case AArch64::ORNXrr:
659 case AArch64::ORNWrr: {
662 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
665 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
669 case AArch64::SUBSXrr:
670 case AArch64::SUBSWrr:
676 case AArch64::SUBXrr:
677 case AArch64::SUBWrr: {
680 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
683 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
689 assert(Opc && SrcOpNum &&
"Missing parameters");
701 int &FalseCycles)
const {
705 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
712 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
716 unsigned ExtraCondLat =
Cond.size() != 1;
720 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
721 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
723 CondCycles = 1 + ExtraCondLat;
724 TrueCycles = FalseCycles = 1;
734 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
735 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
736 CondCycles = 5 + ExtraCondLat;
737 TrueCycles = FalseCycles = 2;
754 switch (
Cond.size()) {
763 switch (
Cond[1].getImm()) {
786 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
792 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
802 switch (
Cond[1].getImm()) {
815 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
831 bool TryFold =
false;
832 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
833 RC = &AArch64::GPR64RegClass;
834 Opc = AArch64::CSELXr;
836 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
837 RC = &AArch64::GPR32RegClass;
838 Opc = AArch64::CSELWr;
840 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
841 RC = &AArch64::FPR64RegClass;
842 Opc = AArch64::FCSELDrrr;
843 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
844 RC = &AArch64::FPR32RegClass;
845 Opc = AArch64::FCSELSrrr;
847 assert(RC &&
"Unsupported regclass");
851 unsigned NewVReg = 0;
866 MRI.clearKillFlags(NewVReg);
871 MRI.constrainRegClass(TrueReg, RC);
872 MRI.constrainRegClass(FalseReg, RC);
887 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
892 return Is.
size() <= 2;
898 if (Subtarget.hasExynosCheapAsMoveHandling()) {
899 if (isExynosCheapAsMove(
MI))
901 return MI.isAsCheapAsAMove();
904 switch (
MI.getOpcode()) {
906 return MI.isAsCheapAsAMove();
908 case AArch64::ADDWrs:
909 case AArch64::ADDXrs:
910 case AArch64::SUBWrs:
911 case AArch64::SUBXrs:
912 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
917 case AArch64::MOVi32imm:
919 case AArch64::MOVi64imm:
925 switch (
MI.getOpcode()) {
929 case AArch64::ADDWrs:
930 case AArch64::ADDXrs:
931 case AArch64::ADDSWrs:
932 case AArch64::ADDSXrs: {
933 unsigned Imm =
MI.getOperand(3).getImm();
940 case AArch64::ADDWrx:
941 case AArch64::ADDXrx:
942 case AArch64::ADDXrx64:
943 case AArch64::ADDSWrx:
944 case AArch64::ADDSXrx:
945 case AArch64::ADDSXrx64: {
946 unsigned Imm =
MI.getOperand(3).getImm();
958 case AArch64::SUBWrs:
959 case AArch64::SUBSWrs: {
960 unsigned Imm =
MI.getOperand(3).getImm();
962 return ShiftVal == 0 ||
966 case AArch64::SUBXrs:
967 case AArch64::SUBSXrs: {
968 unsigned Imm =
MI.getOperand(3).getImm();
970 return ShiftVal == 0 ||
974 case AArch64::SUBWrx:
975 case AArch64::SUBXrx:
976 case AArch64::SUBXrx64:
977 case AArch64::SUBSWrx:
978 case AArch64::SUBSXrx:
979 case AArch64::SUBSXrx64: {
980 unsigned Imm =
MI.getOperand(3).getImm();
992 case AArch64::LDRBBroW:
993 case AArch64::LDRBBroX:
994 case AArch64::LDRBroW:
995 case AArch64::LDRBroX:
996 case AArch64::LDRDroW:
997 case AArch64::LDRDroX:
998 case AArch64::LDRHHroW:
999 case AArch64::LDRHHroX:
1000 case AArch64::LDRHroW:
1001 case AArch64::LDRHroX:
1002 case AArch64::LDRQroW:
1003 case AArch64::LDRQroX:
1004 case AArch64::LDRSBWroW:
1005 case AArch64::LDRSBWroX:
1006 case AArch64::LDRSBXroW:
1007 case AArch64::LDRSBXroX:
1008 case AArch64::LDRSHWroW:
1009 case AArch64::LDRSHWroX:
1010 case AArch64::LDRSHXroW:
1011 case AArch64::LDRSHXroX:
1012 case AArch64::LDRSWroW:
1013 case AArch64::LDRSWroX:
1014 case AArch64::LDRSroW:
1015 case AArch64::LDRSroX:
1016 case AArch64::LDRWroW:
1017 case AArch64::LDRWroX:
1018 case AArch64::LDRXroW:
1019 case AArch64::LDRXroX:
1020 case AArch64::PRFMroW:
1021 case AArch64::PRFMroX:
1022 case AArch64::STRBBroW:
1023 case AArch64::STRBBroX:
1024 case AArch64::STRBroW:
1025 case AArch64::STRBroX:
1026 case AArch64::STRDroW:
1027 case AArch64::STRDroX:
1028 case AArch64::STRHHroW:
1029 case AArch64::STRHHroX:
1030 case AArch64::STRHroW:
1031 case AArch64::STRHroX:
1032 case AArch64::STRQroW:
1033 case AArch64::STRQroX:
1034 case AArch64::STRSroW:
1035 case AArch64::STRSroX:
1036 case AArch64::STRWroW:
1037 case AArch64::STRWroX:
1038 case AArch64::STRXroW:
1039 case AArch64::STRXroX: {
1040 unsigned IsSigned =
MI.getOperand(3).getImm();
1047 unsigned Opc =
MI.getOpcode();
1051 case AArch64::SEH_StackAlloc:
1052 case AArch64::SEH_SaveFPLR:
1053 case AArch64::SEH_SaveFPLR_X:
1054 case AArch64::SEH_SaveReg:
1055 case AArch64::SEH_SaveReg_X:
1056 case AArch64::SEH_SaveRegP:
1057 case AArch64::SEH_SaveRegP_X:
1058 case AArch64::SEH_SaveFReg:
1059 case AArch64::SEH_SaveFReg_X:
1060 case AArch64::SEH_SaveFRegP:
1061 case AArch64::SEH_SaveFRegP_X:
1062 case AArch64::SEH_SetFP:
1063 case AArch64::SEH_AddFP:
1064 case AArch64::SEH_Nop:
1065 case AArch64::SEH_PrologEnd:
1066 case AArch64::SEH_EpilogStart:
1067 case AArch64::SEH_EpilogEnd:
1068 case AArch64::SEH_PACSignLR:
1075 unsigned &SubIdx)
const {
1076 switch (
MI.getOpcode()) {
1079 case AArch64::SBFMXri:
1080 case AArch64::UBFMXri:
1083 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1086 SrcReg =
MI.getOperand(1).getReg();
1087 DstReg =
MI.getOperand(0).getReg();
1088 SubIdx = AArch64::sub_32;
1097 int64_t OffsetA = 0, OffsetB = 0;
1098 unsigned WidthA = 0, WidthB = 0;
1099 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1120 OffsetAIsScalable == OffsetBIsScalable) {
1121 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1122 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1123 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1124 if (LowOffset + LowWidth <= HighOffset)
1136 switch (
MI.getOpcode()) {
1139 if (
MI.getOperand(0).getImm() == 0x14)
1146 case AArch64::MSRpstatesvcrImm1:
1153 auto Next = std::next(
MI.getIterator());
1154 return Next !=
MBB->
end() && Next->isCFIInstruction();
1161 Register &SrcReg2, int64_t &CmpMask,
1162 int64_t &CmpValue)
const {
1165 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1166 if (!
MI.getOperand(1).isReg())
1169 switch (
MI.getOpcode()) {
1172 case AArch64::PTEST_PP:
1173 case AArch64::PTEST_PP_ANY:
1174 SrcReg =
MI.getOperand(0).getReg();
1175 SrcReg2 =
MI.getOperand(1).getReg();
1180 case AArch64::SUBSWrr:
1181 case AArch64::SUBSWrs:
1182 case AArch64::SUBSWrx:
1183 case AArch64::SUBSXrr:
1184 case AArch64::SUBSXrs:
1185 case AArch64::SUBSXrx:
1186 case AArch64::ADDSWrr:
1187 case AArch64::ADDSWrs:
1188 case AArch64::ADDSWrx:
1189 case AArch64::ADDSXrr:
1190 case AArch64::ADDSXrs:
1191 case AArch64::ADDSXrx:
1193 SrcReg =
MI.getOperand(1).getReg();
1194 SrcReg2 =
MI.getOperand(2).getReg();
1198 case AArch64::SUBSWri:
1199 case AArch64::ADDSWri:
1200 case AArch64::SUBSXri:
1201 case AArch64::ADDSXri:
1202 SrcReg =
MI.getOperand(1).getReg();
1205 CmpValue =
MI.getOperand(2).getImm();
1207 case AArch64::ANDSWri:
1208 case AArch64::ANDSXri:
1211 SrcReg =
MI.getOperand(1).getReg();
1215 MI.getOperand(2).getImm(),
1216 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1225 assert(
MBB &&
"Can't get MachineBasicBlock here");
1227 assert(MF &&
"Can't get MachineFunction here");
1232 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1236 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1239 if (!OpRegCstraints)
1247 "Operand has register constraints without being a register!");
1250 if (Reg.isPhysical()) {
1251 if (!OpRegCstraints->
contains(Reg))
1254 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1267 bool MIDefinesZeroReg =
false;
1268 if (
MI.definesRegister(AArch64::WZR) ||
MI.definesRegister(AArch64::XZR))
1269 MIDefinesZeroReg =
true;
1271 switch (
MI.getOpcode()) {
1273 return MI.getOpcode();
1274 case AArch64::ADDSWrr:
1275 return AArch64::ADDWrr;
1276 case AArch64::ADDSWri:
1277 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1278 case AArch64::ADDSWrs:
1279 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1280 case AArch64::ADDSWrx:
1281 return AArch64::ADDWrx;
1282 case AArch64::ADDSXrr:
1283 return AArch64::ADDXrr;
1284 case AArch64::ADDSXri:
1285 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1286 case AArch64::ADDSXrs:
1287 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1288 case AArch64::ADDSXrx:
1289 return AArch64::ADDXrx;
1290 case AArch64::SUBSWrr:
1291 return AArch64::SUBWrr;
1292 case AArch64::SUBSWri:
1293 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1294 case AArch64::SUBSWrs:
1295 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1296 case AArch64::SUBSWrx:
1297 return AArch64::SUBWrx;
1298 case AArch64::SUBSXrr:
1299 return AArch64::SUBXrr;
1300 case AArch64::SUBSXri:
1301 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1302 case AArch64::SUBSXrs:
1303 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1304 case AArch64::SUBSXrx:
1305 return AArch64::SUBXrx;
1320 if (To == To->getParent()->begin())
1325 if (To->getParent() !=
From->getParent())
1337 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1338 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1346bool AArch64InstrInfo::optimizePTestInstr(
1347 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1349 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1350 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1351 auto NewOp = Pred->getOpcode();
1352 bool OpChanged =
false;
1354 unsigned MaskOpcode =
Mask->getOpcode();
1355 unsigned PredOpcode = Pred->getOpcode();
1359 if (
isPTrueOpcode(MaskOpcode) && (PredIsPTestLike || PredIsWhileLike) &&
1362 Mask->getOperand(1).getImm() == 31) {
1371 if (PredIsPTestLike) {
1372 auto PTestLikeMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1373 if (Mask != PTestLikeMask && PTest->
getOpcode() != AArch64::PTEST_PP_ANY)
1378 }
else if ((Mask == Pred) && (PredIsPTestLike || PredIsWhileLike) &&
1379 PTest->
getOpcode() == AArch64::PTEST_PP_ANY) {
1385 }
else if (PredIsPTestLike) {
1408 auto PTestLikeMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1410 if ((Mask != PTestLikeMask) ||
1412 PTest->
getOpcode() != AArch64::PTEST_PP_ANY))
1419 switch (PredOpcode) {
1420 case AArch64::AND_PPzPP:
1421 case AArch64::BIC_PPzPP:
1422 case AArch64::EOR_PPzPP:
1423 case AArch64::NAND_PPzPP:
1424 case AArch64::NOR_PPzPP:
1425 case AArch64::ORN_PPzPP:
1426 case AArch64::ORR_PPzPP:
1427 case AArch64::BRKA_PPzP:
1428 case AArch64::BRKPA_PPzPP:
1429 case AArch64::BRKB_PPzP:
1430 case AArch64::BRKPB_PPzPP:
1431 case AArch64::RDFFR_PPz: {
1434 auto *PredMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1435 if (Mask != PredMask)
1439 case AArch64::BRKN_PPzP: {
1443 if ((MaskOpcode != AArch64::PTRUE_B) ||
1444 (
Mask->getOperand(1).getImm() != 31))
1448 case AArch64::PTRUE_B:
1471 Pred->setDesc(
get(NewOp));
1476 assert(succeeded &&
"Operands have incompatible register classes!");
1477 Pred->addRegisterDefined(AArch64::NZCV,
TRI);
1481 if (Pred->registerDefIsDead(AArch64::NZCV,
TRI)) {
1482 unsigned i = 0,
e = Pred->getNumOperands();
1483 for (; i !=
e; ++i) {
1513 if (DeadNZCVIdx != -1) {
1528 assert(succeeded &&
"Some operands reg class are incompatible!");
1532 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1533 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1534 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1543 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1545 return (CmpValue == 0 || CmpValue == 1) &&
1546 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1554 switch (Instr.getOpcode()) {
1556 return AArch64::INSTRUCTION_LIST_END;
1558 case AArch64::ADDSWrr:
1559 case AArch64::ADDSWri:
1560 case AArch64::ADDSXrr:
1561 case AArch64::ADDSXri:
1562 case AArch64::SUBSWrr:
1563 case AArch64::SUBSWri:
1564 case AArch64::SUBSXrr:
1565 case AArch64::SUBSXri:
1566 return Instr.getOpcode();
1568 case AArch64::ADDWrr:
1569 return AArch64::ADDSWrr;
1570 case AArch64::ADDWri:
1571 return AArch64::ADDSWri;
1572 case AArch64::ADDXrr:
1573 return AArch64::ADDSXrr;
1574 case AArch64::ADDXri:
1575 return AArch64::ADDSXri;
1576 case AArch64::ADCWr:
1577 return AArch64::ADCSWr;
1578 case AArch64::ADCXr:
1579 return AArch64::ADCSXr;
1580 case AArch64::SUBWrr:
1581 return AArch64::SUBSWrr;
1582 case AArch64::SUBWri:
1583 return AArch64::SUBSWri;
1584 case AArch64::SUBXrr:
1585 return AArch64::SUBSXrr;
1586 case AArch64::SUBXri:
1587 return AArch64::SUBSXri;
1588 case AArch64::SBCWr:
1589 return AArch64::SBCSWr;
1590 case AArch64::SBCXr:
1591 return AArch64::SBCSXr;
1592 case AArch64::ANDWri:
1593 return AArch64::ANDSWri;
1594 case AArch64::ANDXri:
1595 return AArch64::ANDSXri;
1602 if (BB->isLiveIn(AArch64::NZCV))
1611 switch (Instr.getOpcode()) {
1615 case AArch64::Bcc: {
1616 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1621 case AArch64::CSINVWr:
1622 case AArch64::CSINVXr:
1623 case AArch64::CSINCWr:
1624 case AArch64::CSINCXr:
1625 case AArch64::CSELWr:
1626 case AArch64::CSELXr:
1627 case AArch64::CSNEGWr:
1628 case AArch64::CSNEGXr:
1629 case AArch64::FCSELSrrr:
1630 case AArch64::FCSELDrrr: {
1631 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1644 Instr.getOperand(CCIdx).getImm())
1697std::optional<UsedNZCV>
1702 if (
MI.getParent() != CmpParent)
1703 return std::nullopt;
1706 return std::nullopt;
1711 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1714 return std::nullopt;
1719 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1722 return NZCVUsedAfterCmp;
1726 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1730 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1752 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1758 "Caller guarantees that CmpInstr compares with constant 0");
1761 if (!NZVCUsed || NZVCUsed->C)
1783bool AArch64InstrInfo::substituteCmpToZero(
1794 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1801 MI->setDesc(
get(NewOpc));
1805 assert(succeeded &&
"Some operands reg class are incompatible!");
1806 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1818 assert((CmpValue == 0 || CmpValue == 1) &&
1819 "Only comparisons to 0 or 1 considered for removal!");
1822 unsigned MIOpc =
MI.getOpcode();
1823 if (MIOpc == AArch64::CSINCWr) {
1824 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1825 MI.getOperand(2).getReg() != AArch64::WZR)
1827 }
else if (MIOpc == AArch64::CSINCXr) {
1828 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1829 MI.getOperand(2).getReg() != AArch64::XZR)
1839 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
true) != -1)
1843 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1845 if (CmpValue && !IsSubsRegImm)
1847 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1852 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1855 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1859 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1862 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1863 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1866 if (MIUsedNZCV.
N && !CmpValue)
1908bool AArch64InstrInfo::removeCmpToZeroOrOne(
1916 bool IsInvertCC =
false;
1926 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1937 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1938 MI.getOpcode() != AArch64::CATCHRET)
1946 if (
MI.getOpcode() == AArch64::CATCHRET) {
1955 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1957 FirstEpilogSEH = std::next(FirstEpilogSEH);
1971 if (M.getStackProtectorGuard() ==
"sysreg") {
1981 int Offset = M.getStackProtectorGuardOffset();
2033 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2042 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2084 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2107 switch (
MI.getOpcode()) {
2110 case AArch64::MOVZWi:
2111 case AArch64::MOVZXi:
2112 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2113 assert(
MI.getDesc().getNumOperands() == 3 &&
2114 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2118 case AArch64::ANDWri:
2119 return MI.getOperand(1).getReg() == AArch64::WZR;
2120 case AArch64::ANDXri:
2121 return MI.getOperand(1).getReg() == AArch64::XZR;
2122 case TargetOpcode::COPY:
2123 return MI.getOperand(1).getReg() == AArch64::WZR;
2131 switch (
MI.getOpcode()) {
2134 case TargetOpcode::COPY: {
2137 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2138 AArch64::GPR64RegClass.
contains(DstReg));
2140 case AArch64::ORRXrs:
2141 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2142 assert(
MI.getDesc().getNumOperands() == 4 &&
2143 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2147 case AArch64::ADDXri:
2148 if (
MI.getOperand(2).getImm() == 0) {
2149 assert(
MI.getDesc().getNumOperands() == 4 &&
2150 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2161 switch (
MI.getOpcode()) {
2164 case TargetOpcode::COPY: {
2166 return AArch64::FPR128RegClass.contains(DstReg);
2168 case AArch64::ORRv16i8:
2169 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2170 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2171 "invalid ORRv16i8 operands");
2180 int &FrameIndex)
const {
2181 switch (
MI.getOpcode()) {
2184 case AArch64::LDRWui:
2185 case AArch64::LDRXui:
2186 case AArch64::LDRBui:
2187 case AArch64::LDRHui:
2188 case AArch64::LDRSui:
2189 case AArch64::LDRDui:
2190 case AArch64::LDRQui:
2191 case AArch64::LDR_PXI:
2192 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2193 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2194 FrameIndex =
MI.getOperand(1).getIndex();
2195 return MI.getOperand(0).getReg();
2204 int &FrameIndex)
const {
2205 switch (
MI.getOpcode()) {
2208 case AArch64::STRWui:
2209 case AArch64::STRXui:
2210 case AArch64::STRBui:
2211 case AArch64::STRHui:
2212 case AArch64::STRSui:
2213 case AArch64::STRDui:
2214 case AArch64::STRQui:
2215 case AArch64::STR_PXI:
2216 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2217 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2218 FrameIndex =
MI.getOperand(1).getIndex();
2219 return MI.getOperand(0).getReg();
2229 return MMO->getFlags() & MOSuppressPair;
2235 if (
MI.memoperands_empty())
2243 return MMO->getFlags() & MOStridedAccess;
2251 case AArch64::STURSi:
2252 case AArch64::STRSpre:
2253 case AArch64::STURDi:
2254 case AArch64::STRDpre:
2255 case AArch64::STURQi:
2256 case AArch64::STRQpre:
2257 case AArch64::STURBBi:
2258 case AArch64::STURHHi:
2259 case AArch64::STURWi:
2260 case AArch64::STRWpre:
2261 case AArch64::STURXi:
2262 case AArch64::STRXpre:
2263 case AArch64::LDURSi:
2264 case AArch64::LDRSpre:
2265 case AArch64::LDURDi:
2266 case AArch64::LDRDpre:
2267 case AArch64::LDURQi:
2268 case AArch64::LDRQpre:
2269 case AArch64::LDURWi:
2270 case AArch64::LDRWpre:
2271 case AArch64::LDURXi:
2272 case AArch64::LDRXpre:
2273 case AArch64::LDRSWpre:
2274 case AArch64::LDURSWi:
2275 case AArch64::LDURHHi:
2276 case AArch64::LDURBBi:
2277 case AArch64::LDURSBWi:
2278 case AArch64::LDURSHWi:
2286 case AArch64::PRFMui:
return AArch64::PRFUMi;
2287 case AArch64::LDRXui:
return AArch64::LDURXi;
2288 case AArch64::LDRWui:
return AArch64::LDURWi;
2289 case AArch64::LDRBui:
return AArch64::LDURBi;
2290 case AArch64::LDRHui:
return AArch64::LDURHi;
2291 case AArch64::LDRSui:
return AArch64::LDURSi;
2292 case AArch64::LDRDui:
return AArch64::LDURDi;
2293 case AArch64::LDRQui:
return AArch64::LDURQi;
2294 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2295 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2296 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2297 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2298 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2299 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2300 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2301 case AArch64::STRXui:
return AArch64::STURXi;
2302 case AArch64::STRWui:
return AArch64::STURWi;
2303 case AArch64::STRBui:
return AArch64::STURBi;
2304 case AArch64::STRHui:
return AArch64::STURHi;
2305 case AArch64::STRSui:
return AArch64::STURSi;
2306 case AArch64::STRDui:
return AArch64::STURDi;
2307 case AArch64::STRQui:
return AArch64::STURQi;
2308 case AArch64::STRBBui:
return AArch64::STURBBi;
2309 case AArch64::STRHHui:
return AArch64::STURHHi;
2317 case AArch64::LDPXi:
2318 case AArch64::LDPDi:
2319 case AArch64::STPXi:
2320 case AArch64::STPDi:
2321 case AArch64::LDNPXi:
2322 case AArch64::LDNPDi:
2323 case AArch64::STNPXi:
2324 case AArch64::STNPDi:
2325 case AArch64::LDPQi:
2326 case AArch64::STPQi:
2327 case AArch64::LDNPQi:
2328 case AArch64::STNPQi:
2329 case AArch64::LDPWi:
2330 case AArch64::LDPSi:
2331 case AArch64::STPWi:
2332 case AArch64::STPSi:
2333 case AArch64::LDNPWi:
2334 case AArch64::LDNPSi:
2335 case AArch64::STNPWi:
2336 case AArch64::STNPSi:
2338 case AArch64::STGPi:
2340 case AArch64::LD1B_IMM:
2341 case AArch64::LD1B_H_IMM:
2342 case AArch64::LD1B_S_IMM:
2343 case AArch64::LD1B_D_IMM:
2344 case AArch64::LD1SB_H_IMM:
2345 case AArch64::LD1SB_S_IMM:
2346 case AArch64::LD1SB_D_IMM:
2347 case AArch64::LD1H_IMM:
2348 case AArch64::LD1H_S_IMM:
2349 case AArch64::LD1H_D_IMM:
2350 case AArch64::LD1SH_S_IMM:
2351 case AArch64::LD1SH_D_IMM:
2352 case AArch64::LD1W_IMM:
2353 case AArch64::LD1W_D_IMM:
2354 case AArch64::LD1SW_D_IMM:
2355 case AArch64::LD1D_IMM:
2357 case AArch64::LD2B_IMM:
2358 case AArch64::LD2H_IMM:
2359 case AArch64::LD2W_IMM:
2360 case AArch64::LD2D_IMM:
2361 case AArch64::LD3B_IMM:
2362 case AArch64::LD3H_IMM:
2363 case AArch64::LD3W_IMM:
2364 case AArch64::LD3D_IMM:
2365 case AArch64::LD4B_IMM:
2366 case AArch64::LD4H_IMM:
2367 case AArch64::LD4W_IMM:
2368 case AArch64::LD4D_IMM:
2370 case AArch64::ST1B_IMM:
2371 case AArch64::ST1B_H_IMM:
2372 case AArch64::ST1B_S_IMM:
2373 case AArch64::ST1B_D_IMM:
2374 case AArch64::ST1H_IMM:
2375 case AArch64::ST1H_S_IMM:
2376 case AArch64::ST1H_D_IMM:
2377 case AArch64::ST1W_IMM:
2378 case AArch64::ST1W_D_IMM:
2379 case AArch64::ST1D_IMM:
2381 case AArch64::ST2B_IMM:
2382 case AArch64::ST2H_IMM:
2383 case AArch64::ST2W_IMM:
2384 case AArch64::ST2D_IMM:
2385 case AArch64::ST3B_IMM:
2386 case AArch64::ST3H_IMM:
2387 case AArch64::ST3W_IMM:
2388 case AArch64::ST3D_IMM:
2389 case AArch64::ST4B_IMM:
2390 case AArch64::ST4H_IMM:
2391 case AArch64::ST4W_IMM:
2392 case AArch64::ST4D_IMM:
2394 case AArch64::LD1RB_IMM:
2395 case AArch64::LD1RB_H_IMM:
2396 case AArch64::LD1RB_S_IMM:
2397 case AArch64::LD1RB_D_IMM:
2398 case AArch64::LD1RSB_H_IMM:
2399 case AArch64::LD1RSB_S_IMM:
2400 case AArch64::LD1RSB_D_IMM:
2401 case AArch64::LD1RH_IMM:
2402 case AArch64::LD1RH_S_IMM:
2403 case AArch64::LD1RH_D_IMM:
2404 case AArch64::LD1RSH_S_IMM:
2405 case AArch64::LD1RSH_D_IMM:
2406 case AArch64::LD1RW_IMM:
2407 case AArch64::LD1RW_D_IMM:
2408 case AArch64::LD1RSW_IMM:
2409 case AArch64::LD1RD_IMM:
2411 case AArch64::LDNT1B_ZRI:
2412 case AArch64::LDNT1H_ZRI:
2413 case AArch64::LDNT1W_ZRI:
2414 case AArch64::LDNT1D_ZRI:
2415 case AArch64::STNT1B_ZRI:
2416 case AArch64::STNT1H_ZRI:
2417 case AArch64::STNT1W_ZRI:
2418 case AArch64::STNT1D_ZRI:
2420 case AArch64::LDNF1B_IMM:
2421 case AArch64::LDNF1B_H_IMM:
2422 case AArch64::LDNF1B_S_IMM:
2423 case AArch64::LDNF1B_D_IMM:
2424 case AArch64::LDNF1SB_H_IMM:
2425 case AArch64::LDNF1SB_S_IMM:
2426 case AArch64::LDNF1SB_D_IMM:
2427 case AArch64::LDNF1H_IMM:
2428 case AArch64::LDNF1H_S_IMM:
2429 case AArch64::LDNF1H_D_IMM:
2430 case AArch64::LDNF1SH_S_IMM:
2431 case AArch64::LDNF1SH_D_IMM:
2432 case AArch64::LDNF1W_IMM:
2433 case AArch64::LDNF1W_D_IMM:
2434 case AArch64::LDNF1SW_D_IMM:
2435 case AArch64::LDNF1D_IMM:
2439 case AArch64::LDR_PXI:
2440 case AArch64::STR_PXI:
2446 switch (
MI.getOpcode()) {
2450 case AArch64::STRSui:
2451 case AArch64::STRDui:
2452 case AArch64::STRQui:
2453 case AArch64::STRXui:
2454 case AArch64::STRWui:
2455 case AArch64::LDRSui:
2456 case AArch64::LDRDui:
2457 case AArch64::LDRQui:
2458 case AArch64::LDRXui:
2459 case AArch64::LDRWui:
2460 case AArch64::LDRSWui:
2462 case AArch64::STURSi:
2463 case AArch64::STRSpre:
2464 case AArch64::STURDi:
2465 case AArch64::STRDpre:
2466 case AArch64::STURQi:
2467 case AArch64::STRQpre:
2468 case AArch64::STURWi:
2469 case AArch64::STRWpre:
2470 case AArch64::STURXi:
2471 case AArch64::STRXpre:
2472 case AArch64::LDURSi:
2473 case AArch64::LDRSpre:
2474 case AArch64::LDURDi:
2475 case AArch64::LDRDpre:
2476 case AArch64::LDURQi:
2477 case AArch64::LDRQpre:
2478 case AArch64::LDURWi:
2479 case AArch64::LDRWpre:
2480 case AArch64::LDURXi:
2481 case AArch64::LDRXpre:
2482 case AArch64::LDURSWi:
2483 case AArch64::LDRSWpre:
2493 case AArch64::ADDWri:
2494 return AArch64::ADDSWri;
2495 case AArch64::ADDWrr:
2496 return AArch64::ADDSWrr;
2497 case AArch64::ADDWrs:
2498 return AArch64::ADDSWrs;
2499 case AArch64::ADDWrx:
2500 return AArch64::ADDSWrx;
2501 case AArch64::ANDWri:
2502 return AArch64::ANDSWri;
2503 case AArch64::ANDWrr:
2504 return AArch64::ANDSWrr;
2505 case AArch64::ANDWrs:
2506 return AArch64::ANDSWrs;
2507 case AArch64::BICWrr:
2508 return AArch64::BICSWrr;
2509 case AArch64::BICWrs:
2510 return AArch64::BICSWrs;
2511 case AArch64::SUBWri:
2512 return AArch64::SUBSWri;
2513 case AArch64::SUBWrr:
2514 return AArch64::SUBSWrr;
2515 case AArch64::SUBWrs:
2516 return AArch64::SUBSWrs;
2517 case AArch64::SUBWrx:
2518 return AArch64::SUBSWrx;
2520 case AArch64::ADDXri:
2521 return AArch64::ADDSXri;
2522 case AArch64::ADDXrr:
2523 return AArch64::ADDSXrr;
2524 case AArch64::ADDXrs:
2525 return AArch64::ADDSXrs;
2526 case AArch64::ADDXrx:
2527 return AArch64::ADDSXrx;
2528 case AArch64::ANDXri:
2529 return AArch64::ANDSXri;
2530 case AArch64::ANDXrr:
2531 return AArch64::ANDSXrr;
2532 case AArch64::ANDXrs:
2533 return AArch64::ANDSXrs;
2534 case AArch64::BICXrr:
2535 return AArch64::BICSXrr;
2536 case AArch64::BICXrs:
2537 return AArch64::BICSXrs;
2538 case AArch64::SUBXri:
2539 return AArch64::SUBSXri;
2540 case AArch64::SUBXrr:
2541 return AArch64::SUBSXrr;
2542 case AArch64::SUBXrs:
2543 return AArch64::SUBSXrs;
2544 case AArch64::SUBXrx:
2545 return AArch64::SUBSXrx;
2547 case AArch64::AND_PPzPP:
2548 return AArch64::ANDS_PPzPP;
2549 case AArch64::BIC_PPzPP:
2550 return AArch64::BICS_PPzPP;
2551 case AArch64::EOR_PPzPP:
2552 return AArch64::EORS_PPzPP;
2553 case AArch64::NAND_PPzPP:
2554 return AArch64::NANDS_PPzPP;
2555 case AArch64::NOR_PPzPP:
2556 return AArch64::NORS_PPzPP;
2557 case AArch64::ORN_PPzPP:
2558 return AArch64::ORNS_PPzPP;
2559 case AArch64::ORR_PPzPP:
2560 return AArch64::ORRS_PPzPP;
2561 case AArch64::BRKA_PPzP:
2562 return AArch64::BRKAS_PPzP;
2563 case AArch64::BRKPA_PPzPP:
2564 return AArch64::BRKPAS_PPzPP;
2565 case AArch64::BRKB_PPzP:
2566 return AArch64::BRKBS_PPzP;
2567 case AArch64::BRKPB_PPzPP:
2568 return AArch64::BRKPBS_PPzPP;
2569 case AArch64::BRKN_PPzP:
2570 return AArch64::BRKNS_PPzP;
2571 case AArch64::RDFFR_PPz:
2572 return AArch64::RDFFRS_PPz;
2573 case AArch64::PTRUE_B:
2574 return AArch64::PTRUES_B;
2585 if (
MI.hasOrderedMemoryRef())
2590 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2591 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2592 "Expected a reg or frame index operand.");
2596 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2598 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2611 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2612 Register BaseReg =
MI.getOperand(1).getReg();
2614 if (
MI.modifiesRegister(BaseReg,
TRI))
2627 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2629 MI.getMF()->getFunction().needsUnwindTableEntry();
2635 if (Subtarget.isPaired128Slow()) {
2636 switch (
MI.getOpcode()) {
2639 case AArch64::LDURQi:
2640 case AArch64::STURQi:
2641 case AArch64::LDRQui:
2642 case AArch64::STRQui:
2652 int64_t &
Offset,
bool &OffsetIsScalable,
unsigned &Width,
2665std::optional<ExtAddrMode>
2670 bool OffsetIsScalable;
2671 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2672 return std::nullopt;
2675 return std::nullopt;
2690 int64_t OffsetScale = 1;
2695 case AArch64::LDURQi:
2696 case AArch64::STURQi:
2700 case AArch64::LDURDi:
2701 case AArch64::STURDi:
2702 case AArch64::LDURXi:
2703 case AArch64::STURXi:
2707 case AArch64::LDURWi:
2708 case AArch64::LDURSWi:
2709 case AArch64::STURWi:
2713 case AArch64::LDURHi:
2714 case AArch64::STURHi:
2715 case AArch64::LDURHHi:
2716 case AArch64::STURHHi:
2717 case AArch64::LDURSHXi:
2718 case AArch64::LDURSHWi:
2722 case AArch64::LDRBroX:
2723 case AArch64::LDRBBroX:
2724 case AArch64::LDRSBXroX:
2725 case AArch64::LDRSBWroX:
2726 case AArch64::STRBroX:
2727 case AArch64::STRBBroX:
2728 case AArch64::LDURBi:
2729 case AArch64::LDURBBi:
2730 case AArch64::LDURSBXi:
2731 case AArch64::LDURSBWi:
2732 case AArch64::STURBi:
2733 case AArch64::STURBBi:
2734 case AArch64::LDRBui:
2735 case AArch64::LDRBBui:
2736 case AArch64::LDRSBXui:
2737 case AArch64::LDRSBWui:
2738 case AArch64::STRBui:
2739 case AArch64::STRBBui:
2743 case AArch64::LDRQroX:
2744 case AArch64::STRQroX:
2745 case AArch64::LDRQui:
2746 case AArch64::STRQui:
2751 case AArch64::LDRDroX:
2752 case AArch64::STRDroX:
2753 case AArch64::LDRXroX:
2754 case AArch64::STRXroX:
2755 case AArch64::LDRDui:
2756 case AArch64::STRDui:
2757 case AArch64::LDRXui:
2758 case AArch64::STRXui:
2763 case AArch64::LDRWroX:
2764 case AArch64::LDRSWroX:
2765 case AArch64::STRWroX:
2766 case AArch64::LDRWui:
2767 case AArch64::LDRSWui:
2768 case AArch64::STRWui:
2773 case AArch64::LDRHroX:
2774 case AArch64::STRHroX:
2775 case AArch64::LDRHHroX:
2776 case AArch64::STRHHroX:
2777 case AArch64::LDRSHXroX:
2778 case AArch64::LDRSHWroX:
2779 case AArch64::LDRHui:
2780 case AArch64::STRHui:
2781 case AArch64::LDRHHui:
2782 case AArch64::STRHHui:
2783 case AArch64::LDRSHXui:
2784 case AArch64::LDRSHWui:
2792 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
2816 case AArch64::SBFMXri:
2829 AM.
Scale = OffsetScale;
2834 case TargetOpcode::SUBREG_TO_REG: {
2847 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
2851 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
2853 DefMI.getOperand(3).getImm() != 0)
2860 AM.
Scale = OffsetScale;
2871 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
2872 int64_t NewOffset) ->
bool {
2873 int64_t MinOffset, MaxOffset;
2890 return OldOffset < MinOffset || OldOffset > MaxOffset ||
2891 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
2893 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
2895 int64_t NewOffset = OldOffset + Disp;
2900 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
2910 auto canFoldAddRegIntoAddrMode =
2927 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
2928 Subtarget.isSTRQroSlow();
2937 case AArch64::ADDXri:
2943 return canFoldAddSubImmIntoAddrMode(Disp);
2945 case AArch64::SUBXri:
2951 return canFoldAddSubImmIntoAddrMode(-Disp);
2953 case AArch64::ADDXrs: {
2963 if ((Shift != 2 && Shift != 3) || !Subtarget.hasAddrLSLFast())
2965 if (avoidSlowSTRQ(MemI))
2968 return canFoldAddRegIntoAddrMode(1ULL << Shift);
2971 case AArch64::ADDXrr:
2979 if (!OptSize && avoidSlowSTRQ(MemI))
2981 return canFoldAddRegIntoAddrMode(1);
2983 case AArch64::ADDXrx:
2991 if (!OptSize && avoidSlowSTRQ(MemI))
3000 return canFoldAddRegIntoAddrMode(
3015 case AArch64::LDURQi:
3016 case AArch64::LDRQui:
3017 return AArch64::LDRQroX;
3018 case AArch64::STURQi:
3019 case AArch64::STRQui:
3020 return AArch64::STRQroX;
3021 case AArch64::LDURDi:
3022 case AArch64::LDRDui:
3023 return AArch64::LDRDroX;
3024 case AArch64::STURDi:
3025 case AArch64::STRDui:
3026 return AArch64::STRDroX;
3027 case AArch64::LDURXi:
3028 case AArch64::LDRXui:
3029 return AArch64::LDRXroX;
3030 case AArch64::STURXi:
3031 case AArch64::STRXui:
3032 return AArch64::STRXroX;
3033 case AArch64::LDURWi:
3034 case AArch64::LDRWui:
3035 return AArch64::LDRWroX;
3036 case AArch64::LDURSWi:
3037 case AArch64::LDRSWui:
3038 return AArch64::LDRSWroX;
3039 case AArch64::STURWi:
3040 case AArch64::STRWui:
3041 return AArch64::STRWroX;
3042 case AArch64::LDURHi:
3043 case AArch64::LDRHui:
3044 return AArch64::LDRHroX;
3045 case AArch64::STURHi:
3046 case AArch64::STRHui:
3047 return AArch64::STRHroX;
3048 case AArch64::LDURHHi:
3049 case AArch64::LDRHHui:
3050 return AArch64::LDRHHroX;
3051 case AArch64::STURHHi:
3052 case AArch64::STRHHui:
3053 return AArch64::STRHHroX;
3054 case AArch64::LDURSHXi:
3055 case AArch64::LDRSHXui:
3056 return AArch64::LDRSHXroX;
3057 case AArch64::LDURSHWi:
3058 case AArch64::LDRSHWui:
3059 return AArch64::LDRSHWroX;
3060 case AArch64::LDURBi:
3061 case AArch64::LDRBui:
3062 return AArch64::LDRBroX;
3063 case AArch64::LDURBBi:
3064 case AArch64::LDRBBui:
3065 return AArch64::LDRBBroX;
3066 case AArch64::LDURSBXi:
3067 case AArch64::LDRSBXui:
3068 return AArch64::LDRSBXroX;
3069 case AArch64::LDURSBWi:
3070 case AArch64::LDRSBWui:
3071 return AArch64::LDRSBWroX;
3072 case AArch64::STURBi:
3073 case AArch64::STRBui:
3074 return AArch64::STRBroX;
3075 case AArch64::STURBBi:
3076 case AArch64::STRBBui:
3077 return AArch64::STRBBroX;
3089 case AArch64::LDURQi:
3091 return AArch64::LDRQui;
3092 case AArch64::STURQi:
3094 return AArch64::STRQui;
3095 case AArch64::LDURDi:
3097 return AArch64::LDRDui;
3098 case AArch64::STURDi:
3100 return AArch64::STRDui;
3101 case AArch64::LDURXi:
3103 return AArch64::LDRXui;
3104 case AArch64::STURXi:
3106 return AArch64::STRXui;
3107 case AArch64::LDURWi:
3109 return AArch64::LDRWui;
3110 case AArch64::LDURSWi:
3112 return AArch64::LDRSWui;
3113 case AArch64::STURWi:
3115 return AArch64::STRWui;
3116 case AArch64::LDURHi:
3118 return AArch64::LDRHui;
3119 case AArch64::STURHi:
3121 return AArch64::STRHui;
3122 case AArch64::LDURHHi:
3124 return AArch64::LDRHHui;
3125 case AArch64::STURHHi:
3127 return AArch64::STRHHui;
3128 case AArch64::LDURSHXi:
3130 return AArch64::LDRSHXui;
3131 case AArch64::LDURSHWi:
3133 return AArch64::LDRSHWui;
3134 case AArch64::LDURBi:
3136 return AArch64::LDRBui;
3137 case AArch64::LDURBBi:
3139 return AArch64::LDRBBui;
3140 case AArch64::LDURSBXi:
3142 return AArch64::LDRSBXui;
3143 case AArch64::LDURSBWi:
3145 return AArch64::LDRSBWui;
3146 case AArch64::STURBi:
3148 return AArch64::STRBui;
3149 case AArch64::STURBBi:
3151 return AArch64::STRBBui;
3152 case AArch64::LDRQui:
3153 case AArch64::STRQui:
3156 case AArch64::LDRDui:
3157 case AArch64::STRDui:
3158 case AArch64::LDRXui:
3159 case AArch64::STRXui:
3162 case AArch64::LDRWui:
3163 case AArch64::LDRSWui:
3164 case AArch64::STRWui:
3167 case AArch64::LDRHui:
3168 case AArch64::STRHui:
3169 case AArch64::LDRHHui:
3170 case AArch64::STRHHui:
3171 case AArch64::LDRSHXui:
3172 case AArch64::LDRSHWui:
3175 case AArch64::LDRBui:
3176 case AArch64::LDRBBui:
3177 case AArch64::LDRSBXui:
3178 case AArch64::LDRSBWui:
3179 case AArch64::STRBui:
3180 case AArch64::STRBBui:
3194 case AArch64::LDURQi:
3195 case AArch64::STURQi:
3196 case AArch64::LDURDi:
3197 case AArch64::STURDi:
3198 case AArch64::LDURXi:
3199 case AArch64::STURXi:
3200 case AArch64::LDURWi:
3201 case AArch64::LDURSWi:
3202 case AArch64::STURWi:
3203 case AArch64::LDURHi:
3204 case AArch64::STURHi:
3205 case AArch64::LDURHHi:
3206 case AArch64::STURHHi:
3207 case AArch64::LDURSHXi:
3208 case AArch64::LDURSHWi:
3209 case AArch64::LDURBi:
3210 case AArch64::STURBi:
3211 case AArch64::LDURBBi:
3212 case AArch64::STURBBi:
3213 case AArch64::LDURSBWi:
3214 case AArch64::LDURSBXi:
3216 case AArch64::LDRQui:
3217 return AArch64::LDURQi;
3218 case AArch64::STRQui:
3219 return AArch64::STURQi;
3220 case AArch64::LDRDui:
3221 return AArch64::LDURDi;
3222 case AArch64::STRDui:
3223 return AArch64::STURDi;
3224 case AArch64::LDRXui:
3225 return AArch64::LDURXi;
3226 case AArch64::STRXui:
3227 return AArch64::STURXi;
3228 case AArch64::LDRWui:
3229 return AArch64::LDURWi;
3230 case AArch64::LDRSWui:
3231 return AArch64::LDURSWi;
3232 case AArch64::STRWui:
3233 return AArch64::STURWi;
3234 case AArch64::LDRHui:
3235 return AArch64::LDURHi;
3236 case AArch64::STRHui:
3237 return AArch64::STURHi;
3238 case AArch64::LDRHHui:
3239 return AArch64::LDURHHi;
3240 case AArch64::STRHHui:
3241 return AArch64::STURHHi;
3242 case AArch64::LDRSHXui:
3243 return AArch64::LDURSHXi;
3244 case AArch64::LDRSHWui:
3245 return AArch64::LDURSHWi;
3246 case AArch64::LDRBBui:
3247 return AArch64::LDURBBi;
3248 case AArch64::LDRBui:
3249 return AArch64::LDURBi;
3250 case AArch64::STRBBui:
3251 return AArch64::STURBBi;
3252 case AArch64::STRBui:
3253 return AArch64::STURBi;
3254 case AArch64::LDRSBWui:
3255 return AArch64::LDURSBWi;
3256 case AArch64::LDRSBXui:
3257 return AArch64::LDURSBXi;
3270 case AArch64::LDRQroX:
3271 case AArch64::LDURQi:
3272 case AArch64::LDRQui:
3273 return AArch64::LDRQroW;
3274 case AArch64::STRQroX:
3275 case AArch64::STURQi:
3276 case AArch64::STRQui:
3277 return AArch64::STRQroW;
3278 case AArch64::LDRDroX:
3279 case AArch64::LDURDi:
3280 case AArch64::LDRDui:
3281 return AArch64::LDRDroW;
3282 case AArch64::STRDroX:
3283 case AArch64::STURDi:
3284 case AArch64::STRDui:
3285 return AArch64::STRDroW;
3286 case AArch64::LDRXroX:
3287 case AArch64::LDURXi:
3288 case AArch64::LDRXui:
3289 return AArch64::LDRXroW;
3290 case AArch64::STRXroX:
3291 case AArch64::STURXi:
3292 case AArch64::STRXui:
3293 return AArch64::STRXroW;
3294 case AArch64::LDRWroX:
3295 case AArch64::LDURWi:
3296 case AArch64::LDRWui:
3297 return AArch64::LDRWroW;
3298 case AArch64::LDRSWroX:
3299 case AArch64::LDURSWi:
3300 case AArch64::LDRSWui:
3301 return AArch64::LDRSWroW;
3302 case AArch64::STRWroX:
3303 case AArch64::STURWi:
3304 case AArch64::STRWui:
3305 return AArch64::STRWroW;
3306 case AArch64::LDRHroX:
3307 case AArch64::LDURHi:
3308 case AArch64::LDRHui:
3309 return AArch64::LDRHroW;
3310 case AArch64::STRHroX:
3311 case AArch64::STURHi:
3312 case AArch64::STRHui:
3313 return AArch64::STRHroW;
3314 case AArch64::LDRHHroX:
3315 case AArch64::LDURHHi:
3316 case AArch64::LDRHHui:
3317 return AArch64::LDRHHroW;
3318 case AArch64::STRHHroX:
3319 case AArch64::STURHHi:
3320 case AArch64::STRHHui:
3321 return AArch64::STRHHroW;
3322 case AArch64::LDRSHXroX:
3323 case AArch64::LDURSHXi:
3324 case AArch64::LDRSHXui:
3325 return AArch64::LDRSHXroW;
3326 case AArch64::LDRSHWroX:
3327 case AArch64::LDURSHWi:
3328 case AArch64::LDRSHWui:
3329 return AArch64::LDRSHWroW;
3330 case AArch64::LDRBroX:
3331 case AArch64::LDURBi:
3332 case AArch64::LDRBui:
3333 return AArch64::LDRBroW;
3334 case AArch64::LDRBBroX:
3335 case AArch64::LDURBBi:
3336 case AArch64::LDRBBui:
3337 return AArch64::LDRBBroW;
3338 case AArch64::LDRSBXroX:
3339 case AArch64::LDURSBXi:
3340 case AArch64::LDRSBXui:
3341 return AArch64::LDRSBXroW;
3342 case AArch64::LDRSBWroX:
3343 case AArch64::LDURSBWi:
3344 case AArch64::LDRSBWui:
3345 return AArch64::LDRSBWroW;
3346 case AArch64::STRBroX:
3347 case AArch64::STURBi:
3348 case AArch64::STRBui:
3349 return AArch64::STRBroW;
3350 case AArch64::STRBBroX:
3351 case AArch64::STURBBi:
3352 case AArch64::STRBBui:
3353 return AArch64::STRBBroW;
3368 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3378 return B.getInstr();
3382 "Addressing mode not supported for folding");
3399 return B.getInstr();
3406 "Address offset can be a register or an immediate, but not both");
3408 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3413 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3427 return B.getInstr();
3431 "Function must not be called with an addressing mode it can't handle");
3436 bool &OffsetIsScalable,
unsigned &Width,
3457 int64_t Dummy1, Dummy2;
3476 if (!BaseOp->
isReg() && !BaseOp->
isFI())
3486 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3491 unsigned &Width, int64_t &MinOffset,
3492 int64_t &MaxOffset) {
3499 MinOffset = MaxOffset = 0;
3501 case AArch64::STRWpost:
3502 case AArch64::LDRWpost:
3508 case AArch64::LDURQi:
3509 case AArch64::STURQi:
3515 case AArch64::PRFUMi:
3516 case AArch64::LDURXi:
3517 case AArch64::LDURDi:
3518 case AArch64::STURXi:
3519 case AArch64::STURDi:
3525 case AArch64::LDURWi:
3526 case AArch64::LDURSi:
3527 case AArch64::LDURSWi:
3528 case AArch64::STURWi:
3529 case AArch64::STURSi:
3535 case AArch64::LDURHi:
3536 case AArch64::LDURHHi:
3537 case AArch64::LDURSHXi:
3538 case AArch64::LDURSHWi:
3539 case AArch64::STURHi:
3540 case AArch64::STURHHi:
3546 case AArch64::LDURBi:
3547 case AArch64::LDURBBi:
3548 case AArch64::LDURSBXi:
3549 case AArch64::LDURSBWi:
3550 case AArch64::STURBi:
3551 case AArch64::STURBBi:
3557 case AArch64::LDPQi:
3558 case AArch64::LDNPQi:
3559 case AArch64::STPQi:
3560 case AArch64::STNPQi:
3566 case AArch64::LDRQui:
3567 case AArch64::STRQui:
3573 case AArch64::LDPXi:
3574 case AArch64::LDPDi:
3575 case AArch64::LDNPXi:
3576 case AArch64::LDNPDi:
3577 case AArch64::STPXi:
3578 case AArch64::STPDi:
3579 case AArch64::STNPXi:
3580 case AArch64::STNPDi:
3586 case AArch64::PRFMui:
3587 case AArch64::LDRXui:
3588 case AArch64::LDRDui:
3589 case AArch64::STRXui:
3590 case AArch64::STRDui:
3596 case AArch64::StoreSwiftAsyncContext:
3603 case AArch64::LDPWi:
3604 case AArch64::LDPSi:
3605 case AArch64::LDNPWi:
3606 case AArch64::LDNPSi:
3607 case AArch64::STPWi:
3608 case AArch64::STPSi:
3609 case AArch64::STNPWi:
3610 case AArch64::STNPSi:
3616 case AArch64::LDRWui:
3617 case AArch64::LDRSui:
3618 case AArch64::LDRSWui:
3619 case AArch64::STRWui:
3620 case AArch64::STRSui:
3626 case AArch64::LDRHui:
3627 case AArch64::LDRHHui:
3628 case AArch64::LDRSHWui:
3629 case AArch64::LDRSHXui:
3630 case AArch64::STRHui:
3631 case AArch64::STRHHui:
3637 case AArch64::LDRBui:
3638 case AArch64::LDRBBui:
3639 case AArch64::LDRSBWui:
3640 case AArch64::LDRSBXui:
3641 case AArch64::STRBui:
3642 case AArch64::STRBBui:
3648 case AArch64::STPXpre:
3649 case AArch64::LDPXpost:
3650 case AArch64::STPDpre:
3651 case AArch64::LDPDpost:
3657 case AArch64::STPQpre:
3658 case AArch64::LDPQpost:
3664 case AArch64::STRXpre:
3665 case AArch64::STRDpre:
3666 case AArch64::LDRXpost:
3667 case AArch64::LDRDpost:
3673 case AArch64::STRQpre:
3674 case AArch64::LDRQpost:
3686 case AArch64::TAGPstack:
3696 case AArch64::STZGi:
3702 case AArch64::STR_ZZZZXI:
3703 case AArch64::LDR_ZZZZXI:
3705 Width = SVEMaxBytesPerVector * 4;
3709 case AArch64::STR_ZZZXI:
3710 case AArch64::LDR_ZZZXI:
3712 Width = SVEMaxBytesPerVector * 3;
3716 case AArch64::STR_ZZXI:
3717 case AArch64::LDR_ZZXI:
3719 Width = SVEMaxBytesPerVector * 2;
3723 case AArch64::LDR_PXI:
3724 case AArch64::STR_PXI:
3726 Width = SVEMaxBytesPerVector / 8;
3730 case AArch64::LDR_ZXI:
3731 case AArch64::STR_ZXI:
3733 Width = SVEMaxBytesPerVector;
3737 case AArch64::LD1B_IMM:
3738 case AArch64::LD1H_IMM:
3739 case AArch64::LD1W_IMM:
3740 case AArch64::LD1D_IMM:
3741 case AArch64::LDNT1B_ZRI:
3742 case AArch64::LDNT1H_ZRI:
3743 case AArch64::LDNT1W_ZRI:
3744 case AArch64::LDNT1D_ZRI:
3745 case AArch64::ST1B_IMM:
3746 case AArch64::ST1H_IMM:
3747 case AArch64::ST1W_IMM:
3748 case AArch64::ST1D_IMM:
3749 case AArch64::STNT1B_ZRI:
3750 case AArch64::STNT1H_ZRI:
3751 case AArch64::STNT1W_ZRI:
3752 case AArch64::STNT1D_ZRI:
3753 case AArch64::LDNF1B_IMM:
3754 case AArch64::LDNF1H_IMM:
3755 case AArch64::LDNF1W_IMM:
3756 case AArch64::LDNF1D_IMM:
3760 Width = SVEMaxBytesPerVector;
3764 case AArch64::LD2B_IMM:
3765 case AArch64::LD2H_IMM:
3766 case AArch64::LD2W_IMM:
3767 case AArch64::LD2D_IMM:
3768 case AArch64::ST2B_IMM:
3769 case AArch64::ST2H_IMM:
3770 case AArch64::ST2W_IMM:
3771 case AArch64::ST2D_IMM:
3773 Width = SVEMaxBytesPerVector * 2;
3777 case AArch64::LD3B_IMM:
3778 case AArch64::LD3H_IMM:
3779 case AArch64::LD3W_IMM:
3780 case AArch64::LD3D_IMM:
3781 case AArch64::ST3B_IMM:
3782 case AArch64::ST3H_IMM:
3783 case AArch64::ST3W_IMM:
3784 case AArch64::ST3D_IMM:
3786 Width = SVEMaxBytesPerVector * 3;
3790 case AArch64::LD4B_IMM:
3791 case AArch64::LD4H_IMM:
3792 case AArch64::LD4W_IMM:
3793 case AArch64::LD4D_IMM:
3794 case AArch64::ST4B_IMM:
3795 case AArch64::ST4H_IMM:
3796 case AArch64::ST4W_IMM:
3797 case AArch64::ST4D_IMM:
3799 Width = SVEMaxBytesPerVector * 4;
3803 case AArch64::LD1B_H_IMM:
3804 case AArch64::LD1SB_H_IMM:
3805 case AArch64::LD1H_S_IMM:
3806 case AArch64::LD1SH_S_IMM:
3807 case AArch64::LD1W_D_IMM:
3808 case AArch64::LD1SW_D_IMM:
3809 case AArch64::ST1B_H_IMM:
3810 case AArch64::ST1H_S_IMM:
3811 case AArch64::ST1W_D_IMM:
3812 case AArch64::LDNF1B_H_IMM:
3813 case AArch64::LDNF1SB_H_IMM:
3814 case AArch64::LDNF1H_S_IMM:
3815 case AArch64::LDNF1SH_S_IMM:
3816 case AArch64::LDNF1W_D_IMM:
3817 case AArch64::LDNF1SW_D_IMM:
3821 Width = SVEMaxBytesPerVector / 2;
3825 case AArch64::LD1B_S_IMM:
3826 case AArch64::LD1SB_S_IMM:
3827 case AArch64::LD1H_D_IMM:
3828 case AArch64::LD1SH_D_IMM:
3829 case AArch64::ST1B_S_IMM:
3830 case AArch64::ST1H_D_IMM:
3831 case AArch64::LDNF1B_S_IMM:
3832 case AArch64::LDNF1SB_S_IMM:
3833 case AArch64::LDNF1H_D_IMM:
3834 case AArch64::LDNF1SH_D_IMM:
3838 Width = SVEMaxBytesPerVector / 4;
3842 case AArch64::LD1B_D_IMM:
3843 case AArch64::LD1SB_D_IMM:
3844 case AArch64::ST1B_D_IMM:
3845 case AArch64::LDNF1B_D_IMM:
3846 case AArch64::LDNF1SB_D_IMM:
3850 Width = SVEMaxBytesPerVector / 8;
3854 case AArch64::ST2Gi:
3855 case AArch64::STZ2Gi:
3861 case AArch64::STGPi:
3867 case AArch64::LD1RB_IMM:
3868 case AArch64::LD1RB_H_IMM:
3869 case AArch64::LD1RB_S_IMM:
3870 case AArch64::LD1RB_D_IMM:
3871 case AArch64::LD1RSB_H_IMM:
3872 case AArch64::LD1RSB_S_IMM:
3873 case AArch64::LD1RSB_D_IMM:
3879 case AArch64::LD1RH_IMM:
3880 case AArch64::LD1RH_S_IMM:
3881 case AArch64::LD1RH_D_IMM:
3882 case AArch64::LD1RSH_S_IMM:
3883 case AArch64::LD1RSH_D_IMM:
3889 case AArch64::LD1RW_IMM:
3890 case AArch64::LD1RW_D_IMM:
3891 case AArch64::LD1RSW_IMM:
3897 case AArch64::LD1RD_IMM:
3913 case AArch64::LDRBBui:
3914 case AArch64::LDURBBi:
3915 case AArch64::LDRSBWui:
3916 case AArch64::LDURSBWi:
3917 case AArch64::STRBBui:
3918 case AArch64::STURBBi:
3920 case AArch64::LDRHHui:
3921 case AArch64::LDURHHi:
3922 case AArch64::LDRSHWui:
3923 case AArch64::LDURSHWi:
3924 case AArch64::STRHHui:
3925 case AArch64::STURHHi:
3927 case AArch64::LDRSui:
3928 case AArch64::LDURSi:
3929 case AArch64::LDRSpre:
3930 case AArch64::LDRSWui:
3931 case AArch64::LDURSWi:
3932 case AArch64::LDRSWpre:
3933 case AArch64::LDRWpre:
3934 case AArch64::LDRWui:
3935 case AArch64::LDURWi:
3936 case AArch64::STRSui:
3937 case AArch64::STURSi:
3938 case AArch64::STRSpre:
3939 case AArch64::STRWui:
3940 case AArch64::STURWi:
3941 case AArch64::STRWpre:
3942 case AArch64::LDPSi:
3943 case AArch64::LDPSWi:
3944 case AArch64::LDPWi:
3945 case AArch64::STPSi:
3946 case AArch64::STPWi:
3948 case AArch64::LDRDui:
3949 case AArch64::LDURDi:
3950 case AArch64::LDRDpre:
3951 case AArch64::LDRXui:
3952 case AArch64::LDURXi:
3953 case AArch64::LDRXpre:
3954 case AArch64::STRDui:
3955 case AArch64::STURDi:
3956 case AArch64::STRDpre:
3957 case AArch64::STRXui:
3958 case AArch64::STURXi:
3959 case AArch64::STRXpre:
3960 case AArch64::LDPDi:
3961 case AArch64::LDPXi:
3962 case AArch64::STPDi:
3963 case AArch64::STPXi:
3965 case AArch64::LDRQui:
3966 case AArch64::LDURQi:
3967 case AArch64::STRQui:
3968 case AArch64::STURQi:
3969 case AArch64::STRQpre:
3970 case AArch64::LDPQi:
3971 case AArch64::LDRQpre:
3972 case AArch64::STPQi:
3974 case AArch64::STZGi:
3975 case AArch64::ST2Gi:
3976 case AArch64::STZ2Gi:
3977 case AArch64::STGPi:
3983 switch (
MI.getOpcode()) {
3986 case AArch64::LDRWpre:
3987 case AArch64::LDRXpre:
3988 case AArch64::LDRSWpre:
3989 case AArch64::LDRSpre:
3990 case AArch64::LDRDpre:
3991 case AArch64::LDRQpre:
3997 switch (
MI.getOpcode()) {
4000 case AArch64::STRWpre:
4001 case AArch64::STRXpre:
4002 case AArch64::STRSpre:
4003 case AArch64::STRDpre:
4004 case AArch64::STRQpre:
4014 switch (
MI.getOpcode()) {
4017 case AArch64::LDPSi:
4018 case AArch64::LDPSWi:
4019 case AArch64::LDPDi:
4020 case AArch64::LDPQi:
4021 case AArch64::LDPWi:
4022 case AArch64::LDPXi:
4023 case AArch64::STPSi:
4024 case AArch64::STPDi:
4025 case AArch64::STPQi:
4026 case AArch64::STPWi:
4027 case AArch64::STPXi:
4028 case AArch64::STGPi:
4037 return MI.getOperand(
Idx);
4045 return MI.getOperand(
Idx);
4050 if (
MI.getParent() ==
nullptr)
4060 auto Reg =
Op.getReg();
4061 if (Reg.isPhysical())
4062 return AArch64::FPR16RegClass.
contains(Reg);
4064 return TRC == &AArch64::FPR16RegClass ||
4065 TRC == &AArch64::FPR16_loRegClass;
4074 auto Reg =
Op.getReg();
4075 if (Reg.isPhysical())
4076 return AArch64::FPR128RegClass.
contains(Reg);
4078 return TRC == &AArch64::FPR128RegClass ||
4079 TRC == &AArch64::FPR128_loRegClass;
4088 auto Reg =
Op.getReg();
4089 if (Reg.isPhysical())
4090 return AArch64::FPR128RegClass.
contains(Reg) ||
4091 AArch64::FPR64RegClass.contains(Reg) ||
4092 AArch64::FPR32RegClass.contains(Reg) ||
4093 AArch64::FPR16RegClass.contains(Reg) ||
4094 AArch64::FPR8RegClass.contains(Reg);
4097 return TRC == &AArch64::FPR128RegClass ||
4098 TRC == &AArch64::FPR128_loRegClass ||
4099 TRC == &AArch64::FPR64RegClass ||
4100 TRC == &AArch64::FPR64_loRegClass ||
4101 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4102 TRC == &AArch64::FPR8RegClass;
4124 if (FirstOpc == SecondOpc)
4130 case AArch64::LDRWui:
4131 case AArch64::LDURWi:
4132 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4133 case AArch64::LDRSWui:
4134 case AArch64::LDURSWi:
4135 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4142 int64_t Offset1,
unsigned Opcode1,
int FI2,
4143 int64_t Offset2,
unsigned Opcode2) {
4149 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4152 if (ObjectOffset1 % Scale1 != 0)
4154 ObjectOffset1 /= Scale1;
4156 if (ObjectOffset2 % Scale2 != 0)
4158 ObjectOffset2 /= Scale2;
4159 ObjectOffset1 += Offset1;
4160 ObjectOffset2 += Offset2;
4161 return ObjectOffset1 + 1 == ObjectOffset2;
4173 unsigned NumBytes)
const {
4183 "Only base registers and frame indices are supported.");
4197 unsigned FirstOpc = FirstLdSt.
getOpcode();
4198 unsigned SecondOpc = SecondLdSt.
getOpcode();
4218 if (Offset1 > 63 || Offset1 < -64)
4223 if (BaseOp1.
isFI()) {
4225 "Caller should have ordered offsets.");
4230 BaseOp2.
getIndex(), Offset2, SecondOpc);
4233 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4235 return Offset1 + 1 == Offset2;
4239 unsigned Reg,
unsigned SubIdx,
4243 return MIB.
addReg(Reg, State);
4246 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4247 return MIB.
addReg(Reg, State, SubIdx);
4254 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4263 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4265 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4266 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4267 unsigned NumRegs = Indices.
size();
4269 int SubReg = 0,
End = NumRegs, Incr = 1;
4287 unsigned SrcReg,
bool KillSrc,
4288 unsigned Opcode,
unsigned ZeroReg,
4291 unsigned NumRegs = Indices.
size();
4294 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4295 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4296 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4297 "GPR reg sequences should not be able to overlap");
4313 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4314 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4317 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4319 if (Subtarget.hasZeroCycleRegMove()) {
4322 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4324 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4340 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4345 if (Subtarget.hasZeroCycleRegMove()) {
4348 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4350 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4370 if (AArch64::PPRRegClass.
contains(DestReg) &&
4371 AArch64::PPRRegClass.
contains(SrcReg)) {
4380 if (AArch64::PNRRegClass.
contains(DestReg) &&
4381 AArch64::PPRRegClass.
contains(SrcReg)) {
4382 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4383 "Unexpected predicate-as-counter register.");
4385 if ((DestReg.
id() - AArch64::PN0) == (SrcReg.
id() - AArch64::P0))
4387 MCRegister PPRDestReg = (DestReg - AArch64::PN0) + AArch64::P0;
4396 if (AArch64::PPRRegClass.
contains(DestReg) &&
4397 AArch64::PNRRegClass.
contains(SrcReg)) {
4398 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4399 "Unexpected predicate-as-counter register.");
4401 if ((DestReg.
id() - AArch64::P0) == (SrcReg.
id() - AArch64::PN0))
4403 MCRegister PNRDestReg = (DestReg - AArch64::P0) + AArch64::PN0;
4404 MCRegister PPRSrcReg = (SrcReg - AArch64::PN0) + AArch64::P0;
4414 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4415 AArch64::ZPRRegClass.
contains(SrcReg)) {
4424 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
4425 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
4426 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
4427 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
4429 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
4436 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
4437 AArch64::ZPR3RegClass.
contains(SrcReg)) {
4439 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4447 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
4448 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
4449 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
4450 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
4452 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4453 AArch64::zsub2, AArch64::zsub3};
4459 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
4460 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
4461 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
4467 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
4481 if (AArch64::DDDDRegClass.
contains(DestReg) &&
4482 AArch64::DDDDRegClass.
contains(SrcReg)) {
4483 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4484 AArch64::dsub2, AArch64::dsub3};
4491 if (AArch64::DDDRegClass.
contains(DestReg) &&
4492 AArch64::DDDRegClass.
contains(SrcReg)) {
4493 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4501 if (AArch64::DDRegClass.
contains(DestReg) &&
4502 AArch64::DDRegClass.
contains(SrcReg)) {
4503 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
4510 if (AArch64::QQQQRegClass.
contains(DestReg) &&
4511 AArch64::QQQQRegClass.
contains(SrcReg)) {
4512 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4513 AArch64::qsub2, AArch64::qsub3};
4520 if (AArch64::QQQRegClass.
contains(DestReg) &&
4521 AArch64::QQQRegClass.
contains(SrcReg)) {
4522 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4530 if (AArch64::QQRegClass.
contains(DestReg) &&
4531 AArch64::QQRegClass.
contains(SrcReg)) {
4532 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
4538 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
4539 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
4540 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
4542 AArch64::XZR, Indices);
4546 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
4547 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
4548 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
4550 AArch64::WZR, Indices);
4554 if (AArch64::FPR128RegClass.
contains(DestReg) &&
4555 AArch64::FPR128RegClass.
contains(SrcReg)) {
4559 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
4560 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
4561 else if (Subtarget.hasNEON())
4580 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4581 AArch64::FPR64RegClass.
contains(SrcReg)) {
4587 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4588 AArch64::FPR32RegClass.
contains(SrcReg)) {
4594 if (AArch64::FPR16RegClass.
contains(DestReg) &&
4595 AArch64::FPR16RegClass.
contains(SrcReg)) {
4597 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
4599 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
4605 if (AArch64::FPR8RegClass.
contains(DestReg) &&
4606 AArch64::FPR8RegClass.
contains(SrcReg)) {
4608 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
4610 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
4617 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4618 AArch64::GPR64RegClass.
contains(SrcReg)) {
4623 if (AArch64::GPR64RegClass.
contains(DestReg) &&
4624 AArch64::FPR64RegClass.
contains(SrcReg)) {
4630 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4631 AArch64::GPR32RegClass.
contains(SrcReg)) {
4636 if (AArch64::GPR32RegClass.
contains(DestReg) &&
4637 AArch64::FPR32RegClass.
contains(SrcReg)) {
4643 if (DestReg == AArch64::NZCV) {
4644 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
4646 .
addImm(AArch64SysReg::NZCV)
4652 if (SrcReg == AArch64::NZCV) {
4653 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
4655 .
addImm(AArch64SysReg::NZCV)
4662 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
4663 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
4673 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
4678 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
4680 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
4693 Register SrcReg,
bool isKill,
int FI,
4708 switch (
TRI->getSpillSize(*RC)) {
4710 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4711 Opc = AArch64::STRBui;
4714 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4715 Opc = AArch64::STRHui;
4716 else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
4718 "Unexpected register store without SVE store instructions");
4719 Opc = AArch64::STR_PXI;
4721 }
else if (AArch64::PNRRegClass.hasSubClassEq(RC)) {
4722 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4723 "Unexpected register store without SVE2p1 or SME2");
4724 SrcReg = (SrcReg - AArch64::PN0) + AArch64::P0;
4725 Opc = AArch64::STR_PXI;
4730 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4731 Opc = AArch64::STRWui;
4735 assert(SrcReg != AArch64::WSP);
4736 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4737 Opc = AArch64::STRSui;
4740 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4741 Opc = AArch64::STRXui;
4745 assert(SrcReg != AArch64::SP);
4746 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4747 Opc = AArch64::STRDui;
4748 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4750 get(AArch64::STPWi), SrcReg, isKill,
4751 AArch64::sube32, AArch64::subo32, FI, MMO);
4756 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4757 Opc = AArch64::STRQui;
4758 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4759 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4760 Opc = AArch64::ST1Twov1d;
4762 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4764 get(AArch64::STPXi), SrcReg, isKill,
4765 AArch64::sube64, AArch64::subo64, FI, MMO);
4767 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4769 "Unexpected register store without SVE store instructions");
4770 Opc = AArch64::STR_ZXI;
4775 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4776 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4777 Opc = AArch64::ST1Threev1d;
4782 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4783 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4784 Opc = AArch64::ST1Fourv1d;
4786 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4787 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4788 Opc = AArch64::ST1Twov2d;
4790 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
4791 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4793 "Unexpected register store without SVE store instructions");
4794 Opc = AArch64::STR_ZZXI;
4799 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4800 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4801 Opc = AArch64::ST1Threev2d;
4803 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4805 "Unexpected register store without SVE store instructions");
4806 Opc = AArch64::STR_ZZZXI;
4811 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4812 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4813 Opc = AArch64::ST1Fourv2d;
4815 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
4816 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4818 "Unexpected register store without SVE store instructions");
4819 Opc = AArch64::STR_ZZZZXI;
4824 assert(Opc &&
"Unknown register class");
4835 MI.addMemOperand(MMO);
4842 Register DestReg,
unsigned SubIdx0,
4843 unsigned SubIdx1,
int FI,
4847 bool IsUndef =
true;
4849 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
4851 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
4880 switch (
TRI->getSpillSize(*RC)) {
4882 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4883 Opc = AArch64::LDRBui;
4886 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4887 Opc = AArch64::LDRHui;
4888 else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
4890 "Unexpected register load without SVE load instructions");
4891 Opc = AArch64::LDR_PXI;
4893 }
else if (AArch64::PNRRegClass.hasSubClassEq(RC)) {
4894 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4895 "Unexpected register load without SVE2p1 or SME2");
4897 DestReg = (DestReg - AArch64::PN0) + AArch64::P0;
4898 Opc = AArch64::LDR_PXI;
4903 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4904 Opc = AArch64::LDRWui;
4908 assert(DestReg != AArch64::WSP);
4909 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4910 Opc = AArch64::LDRSui;
4913 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4914 Opc = AArch64::LDRXui;
4918 assert(DestReg != AArch64::SP);
4919 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4920 Opc = AArch64::LDRDui;
4921 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4923 get(AArch64::LDPWi), DestReg, AArch64::sube32,
4924 AArch64::subo32, FI, MMO);
4929 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4930 Opc = AArch64::LDRQui;
4931 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4932 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
4933 Opc = AArch64::LD1Twov1d;
4935 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4937 get(AArch64::LDPXi), DestReg, AArch64::sube64,
4938 AArch64::subo64, FI, MMO);
4940 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4942 "Unexpected register load without SVE load instructions");
4943 Opc = AArch64::LDR_ZXI;
4948 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4949 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
4950 Opc = AArch64::LD1Threev1d;
4955 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4956 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
4957 Opc = AArch64::LD1Fourv1d;
4959 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4960 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
4961 Opc = AArch64::LD1Twov2d;
4963 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
4964 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4966 "Unexpected register load without SVE load instructions");
4967 Opc = AArch64::LDR_ZZXI;
4972 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4973 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
4974 Opc = AArch64::LD1Threev2d;
4976 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4978 "Unexpected register load without SVE load instructions");
4979 Opc = AArch64::LDR_ZZZXI;
4984 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4985 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
4986 Opc = AArch64::LD1Fourv2d;
4988 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
4989 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4991 "Unexpected register load without SVE load instructions");
4992 Opc = AArch64::LDR_ZZZZXI;
4998 assert(Opc &&
"Unknown register class");
5008 MI.addMemOperand(MMO);
5015 UseMI.getIterator()),
5017 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5018 I.readsRegister(AArch64::NZCV, TRI);
5027 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5034 ByteSized =
Offset.getFixed();
5035 VGSized =
Offset.getScalable() / 2;
5043 int64_t &NumDataVectors) {
5047 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5049 NumBytes =
Offset.getFixed();
5051 NumPredicateVectors =
Offset.getScalable() / 2;
5056 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5057 NumPredicateVectors > 62) {
5058 NumDataVectors = NumPredicateVectors / 8;
5059 NumPredicateVectors -= NumDataVectors * 8;
5066 int NumVGScaledBytes,
unsigned VG,
5073 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5074 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5077 if (NumVGScaledBytes) {
5078 Expr.
push_back((uint8_t)dwarf::DW_OP_consts);
5081 Expr.
push_back((uint8_t)dwarf::DW_OP_bregx);
5085 Expr.
push_back((uint8_t)dwarf::DW_OP_mul);
5086 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5088 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5089 << std::abs(NumVGScaledBytes) <<
" * VG";
5098 int64_t NumBytes, NumVGScaledBytes;
5101 std::string CommentBuffer;
5104 if (Reg == AArch64::SP)
5106 else if (Reg == AArch64::FP)