65#define GET_INSTRINFO_CTOR_DTOR
66#include "AArch64GenInstrInfo.inc"
70 cl::desc(
"Restrict range of CB instructions (DEBUG)"));
74 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
78 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
82 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
86 cl::desc(
"Restrict range of B instructions (DEBUG)"));
90 cl::desc(
"Restrict range of instructions to search for the "
91 "machine-combiner gather pattern optimization"));
96 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
107 auto Op =
MI.getOpcode();
108 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
109 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
113 if (
MI.isMetaInstruction())
118 unsigned NumBytes = 0;
122 NumBytes =
Desc.getSize() ?
Desc.getSize() : 4;
125 if (!MFI->shouldSignReturnAddress(MF))
129 auto Method = STI.getAuthenticatedLRCheckMethod(*MF);
137 switch (
Desc.getOpcode()) {
140 return Desc.getSize();
147 case TargetOpcode::STACKMAP:
150 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
152 case TargetOpcode::PATCHPOINT:
155 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
157 case TargetOpcode::STATEPOINT:
159 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
164 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
169 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
171 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
172 case TargetOpcode::PATCHABLE_TAIL_CALL:
173 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
177 case TargetOpcode::PATCHABLE_EVENT_CALL:
183 NumBytes =
MI.getOperand(1).getImm();
185 case TargetOpcode::BUNDLE:
186 NumBytes = getInstBundleLength(
MI);
193unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
197 while (++
I != E &&
I->isInsideBundle()) {
198 assert(!
I->isBundle() &&
"No nested bundle!");
233 case AArch64::CBWPri:
234 case AArch64::CBXPri:
235 case AArch64::CBWPrr:
236 case AArch64::CBXPrr:
265 case AArch64::CBWPri:
266 case AArch64::CBXPri:
267 case AArch64::CBWPrr:
268 case AArch64::CBXPrr:
274 int64_t BrOffset)
const {
276 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
277 "over conditional branch expansion");
278 return isIntN(Bits, BrOffset / 4);
283 switch (
MI.getOpcode()) {
287 return MI.getOperand(0).getMBB();
292 return MI.getOperand(2).getMBB();
298 return MI.getOperand(1).getMBB();
299 case AArch64::CBWPri:
300 case AArch64::CBXPri:
301 case AArch64::CBWPrr:
302 case AArch64::CBXPrr:
303 return MI.getOperand(3).getMBB();
313 assert(RS &&
"RegScavenger required for long branching");
315 "new block should be inserted for expanding unconditional branch");
318 "restore block should be inserted for restoring clobbered registers");
325 "Branch offsets outside of the signed 33-bit range not supported");
336 RS->enterBasicBlockEnd(
MBB);
339 constexpr Register Reg = AArch64::X16;
340 if (!RS->isRegUsed(Reg)) {
341 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
348 Register Scavenged = RS->FindUnusedReg(&AArch64::GPR64RegClass);
349 if (Scavenged != AArch64::NoRegister &&
351 buildIndirectBranch(Scavenged, NewDestBB);
352 RS->setRegUsed(Scavenged);
361 "Unable to insert indirect branch inside function that has red zone");
384 bool AllowModify)
const {
391 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
392 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
396 if (!isUnpredicatedTerminator(*
I))
403 unsigned LastOpc = LastInst->
getOpcode();
404 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
419 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
426 LastInst = SecondLastInst;
428 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
433 SecondLastInst = &*
I;
434 SecondLastOpc = SecondLastInst->
getOpcode();
445 LastInst = SecondLastInst;
447 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
449 "unreachable unconditional branches removed above");
458 SecondLastInst = &*
I;
459 SecondLastOpc = SecondLastInst->
getOpcode();
463 if (SecondLastInst &&
I !=
MBB.begin() && isUnpredicatedTerminator(*--
I))
479 I->eraseFromParent();
488 I->eraseFromParent();
497 MachineBranchPredicate &MBP,
498 bool AllowModify)
const {
508 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
509 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
513 if (!isUnpredicatedTerminator(*
I))
518 unsigned LastOpc = LastInst->
getOpcode();
533 assert(MBP.TrueDest &&
"expected!");
534 MBP.FalseDest =
MBB.getNextNode();
536 MBP.ConditionDef =
nullptr;
537 MBP.SingleUseCondition =
false;
541 MBP.Predicate = (LastOpc == AArch64::CBNZX || LastOpc == AArch64::CBNZW)
542 ? MachineBranchPredicate::PRED_NE
543 : MachineBranchPredicate::PRED_EQ;
559 Cond[1].setImm(AArch64::CBNZW);
562 Cond[1].setImm(AArch64::CBZW);
565 Cond[1].setImm(AArch64::CBNZX);
568 Cond[1].setImm(AArch64::CBZX);
571 Cond[1].setImm(AArch64::TBNZW);
574 Cond[1].setImm(AArch64::TBZW);
577 Cond[1].setImm(AArch64::TBNZX);
580 Cond[1].setImm(AArch64::TBZX);
584 case AArch64::CBWPri:
585 case AArch64::CBXPri:
586 case AArch64::CBWPrr:
587 case AArch64::CBXPrr: {
600 int *BytesRemoved)
const {
610 I->eraseFromParent();
614 if (
I ==
MBB.begin()) {
627 I->eraseFromParent();
634void AArch64InstrInfo::instantiateCondBranch(
664 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
692 if (!
DefMI->isFullCopy())
694 VReg =
DefMI->getOperand(1).getReg();
703 unsigned *NewVReg =
nullptr) {
708 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
711 unsigned SrcOpNum = 0;
712 switch (
DefMI->getOpcode()) {
713 case AArch64::ADDSXri:
714 case AArch64::ADDSWri:
716 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
721 case AArch64::ADDXri:
722 case AArch64::ADDWri:
724 if (!
DefMI->getOperand(2).isImm() ||
DefMI->getOperand(2).getImm() != 1 ||
725 DefMI->getOperand(3).getImm() != 0)
728 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
731 case AArch64::ORNXrr:
732 case AArch64::ORNWrr: {
735 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
738 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
742 case AArch64::SUBSXrr:
743 case AArch64::SUBSWrr:
745 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
750 case AArch64::SUBXrr:
751 case AArch64::SUBWrr: {
754 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
757 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
763 assert(
Opc && SrcOpNum &&
"Missing parameters");
766 *NewVReg =
DefMI->getOperand(SrcOpNum).getReg();
775 int &FalseCycles)
const {
779 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
786 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
790 unsigned ExtraCondLat =
Cond.size() != 1;
794 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
795 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
797 CondCycles = 1 + ExtraCondLat;
798 TrueCycles = FalseCycles = 1;
808 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
809 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
810 CondCycles = 5 + ExtraCondLat;
811 TrueCycles = FalseCycles = 2;
828 switch (
Cond.size()) {
860 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
866 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
905 unsigned SUBSOpC, SUBSDestReg;
911 case AArch64::CBWPri:
912 SUBSOpC = AArch64::SUBSWri;
913 SUBSDestReg = AArch64::WZR;
916 case AArch64::CBXPri:
917 SUBSOpC = AArch64::SUBSXri;
918 SUBSDestReg = AArch64::XZR;
921 case AArch64::CBWPrr:
922 SUBSOpC = AArch64::SUBSWrr;
923 SUBSDestReg = AArch64::WZR;
926 case AArch64::CBXPrr:
927 SUBSOpC = AArch64::SUBSXrr;
928 SUBSDestReg = AArch64::XZR;
947 bool TryFold =
false;
948 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
949 RC = &AArch64::GPR64RegClass;
950 Opc = AArch64::CSELXr;
952 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
953 RC = &AArch64::GPR32RegClass;
954 Opc = AArch64::CSELWr;
956 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
957 RC = &AArch64::FPR64RegClass;
958 Opc = AArch64::FCSELDrrr;
959 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
960 RC = &AArch64::FPR32RegClass;
961 Opc = AArch64::FCSELSrrr;
963 assert(RC &&
"Unsupported regclass");
967 unsigned NewVReg = 0;
982 MRI.clearKillFlags(NewVReg);
987 MRI.constrainRegClass(TrueReg, RC);
988 MRI.constrainRegClass(FalseReg, RC);
1003 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
1008 return Is.
size() <= 2;
1014 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1015 if (isExynosCheapAsMove(
MI))
1017 return MI.isAsCheapAsAMove();
1020 switch (
MI.getOpcode()) {
1022 return MI.isAsCheapAsAMove();
1024 case AArch64::ADDWrs:
1025 case AArch64::ADDXrs:
1026 case AArch64::SUBWrs:
1027 case AArch64::SUBXrs:
1028 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
1033 case AArch64::MOVi32imm:
1035 case AArch64::MOVi64imm:
1040bool AArch64InstrInfo::isFalkorShiftExtFast(
const MachineInstr &
MI) {
1041 switch (
MI.getOpcode()) {
1045 case AArch64::ADDWrs:
1046 case AArch64::ADDXrs:
1047 case AArch64::ADDSWrs:
1048 case AArch64::ADDSXrs: {
1049 unsigned Imm =
MI.getOperand(3).getImm();
1056 case AArch64::ADDWrx:
1057 case AArch64::ADDXrx:
1058 case AArch64::ADDXrx64:
1059 case AArch64::ADDSWrx:
1060 case AArch64::ADDSXrx:
1061 case AArch64::ADDSXrx64: {
1062 unsigned Imm =
MI.getOperand(3).getImm();
1074 case AArch64::SUBWrs:
1075 case AArch64::SUBSWrs: {
1076 unsigned Imm =
MI.getOperand(3).getImm();
1078 return ShiftVal == 0 ||
1082 case AArch64::SUBXrs:
1083 case AArch64::SUBSXrs: {
1084 unsigned Imm =
MI.getOperand(3).getImm();
1086 return ShiftVal == 0 ||
1090 case AArch64::SUBWrx:
1091 case AArch64::SUBXrx:
1092 case AArch64::SUBXrx64:
1093 case AArch64::SUBSWrx:
1094 case AArch64::SUBSXrx:
1095 case AArch64::SUBSXrx64: {
1096 unsigned Imm =
MI.getOperand(3).getImm();
1108 case AArch64::LDRBBroW:
1109 case AArch64::LDRBBroX:
1110 case AArch64::LDRBroW:
1111 case AArch64::LDRBroX:
1112 case AArch64::LDRDroW:
1113 case AArch64::LDRDroX:
1114 case AArch64::LDRHHroW:
1115 case AArch64::LDRHHroX:
1116 case AArch64::LDRHroW:
1117 case AArch64::LDRHroX:
1118 case AArch64::LDRQroW:
1119 case AArch64::LDRQroX:
1120 case AArch64::LDRSBWroW:
1121 case AArch64::LDRSBWroX:
1122 case AArch64::LDRSBXroW:
1123 case AArch64::LDRSBXroX:
1124 case AArch64::LDRSHWroW:
1125 case AArch64::LDRSHWroX:
1126 case AArch64::LDRSHXroW:
1127 case AArch64::LDRSHXroX:
1128 case AArch64::LDRSWroW:
1129 case AArch64::LDRSWroX:
1130 case AArch64::LDRSroW:
1131 case AArch64::LDRSroX:
1132 case AArch64::LDRWroW:
1133 case AArch64::LDRWroX:
1134 case AArch64::LDRXroW:
1135 case AArch64::LDRXroX:
1136 case AArch64::PRFMroW:
1137 case AArch64::PRFMroX:
1138 case AArch64::STRBBroW:
1139 case AArch64::STRBBroX:
1140 case AArch64::STRBroW:
1141 case AArch64::STRBroX:
1142 case AArch64::STRDroW:
1143 case AArch64::STRDroX:
1144 case AArch64::STRHHroW:
1145 case AArch64::STRHHroX:
1146 case AArch64::STRHroW:
1147 case AArch64::STRHroX:
1148 case AArch64::STRQroW:
1149 case AArch64::STRQroX:
1150 case AArch64::STRSroW:
1151 case AArch64::STRSroX:
1152 case AArch64::STRWroW:
1153 case AArch64::STRWroX:
1154 case AArch64::STRXroW:
1155 case AArch64::STRXroX: {
1156 unsigned IsSigned =
MI.getOperand(3).getImm();
1163 unsigned Opc =
MI.getOpcode();
1167 case AArch64::SEH_StackAlloc:
1168 case AArch64::SEH_SaveFPLR:
1169 case AArch64::SEH_SaveFPLR_X:
1170 case AArch64::SEH_SaveReg:
1171 case AArch64::SEH_SaveReg_X:
1172 case AArch64::SEH_SaveRegP:
1173 case AArch64::SEH_SaveRegP_X:
1174 case AArch64::SEH_SaveFReg:
1175 case AArch64::SEH_SaveFReg_X:
1176 case AArch64::SEH_SaveFRegP:
1177 case AArch64::SEH_SaveFRegP_X:
1178 case AArch64::SEH_SetFP:
1179 case AArch64::SEH_AddFP:
1180 case AArch64::SEH_Nop:
1181 case AArch64::SEH_PrologEnd:
1182 case AArch64::SEH_EpilogStart:
1183 case AArch64::SEH_EpilogEnd:
1184 case AArch64::SEH_PACSignLR:
1185 case AArch64::SEH_SaveAnyRegQP:
1186 case AArch64::SEH_SaveAnyRegQPX:
1187 case AArch64::SEH_AllocZ:
1188 case AArch64::SEH_SaveZReg:
1189 case AArch64::SEH_SavePReg:
1196 unsigned &SubIdx)
const {
1197 switch (
MI.getOpcode()) {
1200 case AArch64::SBFMXri:
1201 case AArch64::UBFMXri:
1204 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1207 SrcReg =
MI.getOperand(1).getReg();
1208 DstReg =
MI.getOperand(0).getReg();
1209 SubIdx = AArch64::sub_32;
1218 int64_t OffsetA = 0, OffsetB = 0;
1219 TypeSize WidthA(0,
false), WidthB(0,
false);
1220 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1241 OffsetAIsScalable == OffsetBIsScalable) {
1242 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1243 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1244 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1245 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1263 switch (
MI.getOpcode()) {
1266 if (
MI.getOperand(0).getImm() == 0x14)
1273 case AArch64::MSRpstatesvcrImm1:
1280 auto Next = std::next(
MI.getIterator());
1281 return Next !=
MBB->end() &&
Next->isCFIInstruction();
1288 Register &SrcReg2, int64_t &CmpMask,
1289 int64_t &CmpValue)
const {
1293 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1294 if (!
MI.getOperand(1).isReg() ||
MI.getOperand(1).getSubReg())
1297 switch (
MI.getOpcode()) {
1300 case AArch64::PTEST_PP:
1301 case AArch64::PTEST_PP_ANY:
1302 case AArch64::PTEST_PP_FIRST:
1303 SrcReg =
MI.getOperand(0).getReg();
1304 SrcReg2 =
MI.getOperand(1).getReg();
1305 if (
MI.getOperand(2).getSubReg())
1312 case AArch64::SUBSWrr:
1313 case AArch64::SUBSWrs:
1314 case AArch64::SUBSWrx:
1315 case AArch64::SUBSXrr:
1316 case AArch64::SUBSXrs:
1317 case AArch64::SUBSXrx:
1318 case AArch64::ADDSWrr:
1319 case AArch64::ADDSWrs:
1320 case AArch64::ADDSWrx:
1321 case AArch64::ADDSXrr:
1322 case AArch64::ADDSXrs:
1323 case AArch64::ADDSXrx:
1325 SrcReg =
MI.getOperand(1).getReg();
1326 SrcReg2 =
MI.getOperand(2).getReg();
1329 if (
MI.getOperand(2).getSubReg())
1335 case AArch64::SUBSWri:
1336 case AArch64::ADDSWri:
1337 case AArch64::SUBSXri:
1338 case AArch64::ADDSXri:
1339 SrcReg =
MI.getOperand(1).getReg();
1342 CmpValue =
MI.getOperand(2).getImm();
1344 case AArch64::ANDSWri:
1345 case AArch64::ANDSXri:
1348 SrcReg =
MI.getOperand(1).getReg();
1352 MI.getOperand(2).getImm(),
1353 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1362 assert(
MBB &&
"Can't get MachineBasicBlock here");
1364 assert(MF &&
"Can't get MachineFunction here");
1369 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands();
OpIdx < EndIdx;
1376 if (!OpRegCstraints)
1384 "Operand has register constraints without being a register!");
1387 if (
Reg.isPhysical()) {
1391 !
MRI->constrainRegClass(
Reg, OpRegCstraints))
1404 bool MIDefinesZeroReg =
false;
1405 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1406 MI.definesRegister(AArch64::XZR,
nullptr))
1407 MIDefinesZeroReg =
true;
1409 switch (
MI.getOpcode()) {
1411 return MI.getOpcode();
1412 case AArch64::ADDSWrr:
1413 return AArch64::ADDWrr;
1414 case AArch64::ADDSWri:
1415 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1416 case AArch64::ADDSWrs:
1417 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1418 case AArch64::ADDSWrx:
1419 return AArch64::ADDWrx;
1420 case AArch64::ADDSXrr:
1421 return AArch64::ADDXrr;
1422 case AArch64::ADDSXri:
1423 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1424 case AArch64::ADDSXrs:
1425 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1426 case AArch64::ADDSXrx:
1427 return AArch64::ADDXrx;
1428 case AArch64::SUBSWrr:
1429 return AArch64::SUBWrr;
1430 case AArch64::SUBSWri:
1431 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1432 case AArch64::SUBSWrs:
1433 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1434 case AArch64::SUBSWrx:
1435 return AArch64::SUBWrx;
1436 case AArch64::SUBSXrr:
1437 return AArch64::SUBXrr;
1438 case AArch64::SUBSXri:
1439 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1440 case AArch64::SUBSXrs:
1441 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1442 case AArch64::SUBSXrx:
1443 return AArch64::SUBXrx;
1458 if (To == To->getParent()->begin())
1463 if (To->getParent() != From->getParent())
1475 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1476 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1482std::optional<unsigned>
1486 unsigned MaskOpcode =
Mask->getOpcode();
1487 unsigned PredOpcode = Pred->
getOpcode();
1488 bool PredIsPTestLike = isPTestLikeOpcode(PredOpcode);
1489 bool PredIsWhileLike = isWhileOpcode(PredOpcode);
1491 if (PredIsWhileLike) {
1495 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1502 getElementSizeForOpcode(MaskOpcode) ==
1503 getElementSizeForOpcode(PredOpcode))
1509 if (PTest->
getOpcode() == AArch64::PTEST_PP_FIRST &&
1516 if (PredIsPTestLike) {
1521 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1529 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1530 PTestLikeMask->getOperand(1).getReg().isVirtual())
1532 MRI->getUniqueVRegDef(PTestLikeMask->getOperand(1).getReg());
1538 getElementSizeForOpcode(MaskOpcode) ==
1539 getElementSizeForOpcode(PredOpcode)) {
1540 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1566 uint64_t PredElementSize = getElementSizeForOpcode(PredOpcode);
1568 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1576 switch (PredOpcode) {
1577 case AArch64::AND_PPzPP:
1578 case AArch64::BIC_PPzPP:
1579 case AArch64::EOR_PPzPP:
1580 case AArch64::NAND_PPzPP:
1581 case AArch64::NOR_PPzPP:
1582 case AArch64::ORN_PPzPP:
1583 case AArch64::ORR_PPzPP:
1584 case AArch64::BRKA_PPzP:
1585 case AArch64::BRKPA_PPzPP:
1586 case AArch64::BRKB_PPzP:
1587 case AArch64::BRKPB_PPzPP:
1588 case AArch64::RDFFR_PPz: {
1592 if (Mask != PredMask)
1596 case AArch64::BRKN_PPzP: {
1600 if ((MaskOpcode != AArch64::PTRUE_B) ||
1601 (
Mask->getOperand(1).getImm() != 31))
1605 case AArch64::PTRUE_B:
1618bool AArch64InstrInfo::optimizePTestInstr(
1619 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1621 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1622 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1624 if (Pred->
isCopy() && PTest->
getOpcode() == AArch64::PTEST_PP_FIRST) {
1628 if (
Op.isReg() &&
Op.getReg().isVirtual() &&
1629 Op.getSubReg() == AArch64::psub0)
1630 Pred =
MRI->getUniqueVRegDef(
Op.getReg());
1633 unsigned PredOpcode = Pred->
getOpcode();
1634 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1650 if (*NewOp != PredOpcode) {
1661 for (; i !=
e; ++i) {
1692 if (DeadNZCVIdx != -1) {
1711 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1712 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY ||
1713 CmpInstr.
getOpcode() == AArch64::PTEST_PP_FIRST)
1714 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1723 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1725 return (CmpValue == 0 || CmpValue == 1) &&
1726 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1734 switch (Instr.getOpcode()) {
1736 return AArch64::INSTRUCTION_LIST_END;
1738 case AArch64::ADDSWrr:
1739 case AArch64::ADDSWri:
1740 case AArch64::ADDSXrr:
1741 case AArch64::ADDSXri:
1742 case AArch64::SUBSWrr:
1743 case AArch64::SUBSWri:
1744 case AArch64::SUBSXrr:
1745 case AArch64::SUBSXri:
1746 return Instr.getOpcode();
1748 case AArch64::ADDWrr:
1749 return AArch64::ADDSWrr;
1750 case AArch64::ADDWri:
1751 return AArch64::ADDSWri;
1752 case AArch64::ADDXrr:
1753 return AArch64::ADDSXrr;
1754 case AArch64::ADDXri:
1755 return AArch64::ADDSXri;
1756 case AArch64::ADCWr:
1757 return AArch64::ADCSWr;
1758 case AArch64::ADCXr:
1759 return AArch64::ADCSXr;
1760 case AArch64::SUBWrr:
1761 return AArch64::SUBSWrr;
1762 case AArch64::SUBWri:
1763 return AArch64::SUBSWri;
1764 case AArch64::SUBXrr:
1765 return AArch64::SUBSXrr;
1766 case AArch64::SUBXri:
1767 return AArch64::SUBSXri;
1768 case AArch64::SBCWr:
1769 return AArch64::SBCSWr;
1770 case AArch64::SBCXr:
1771 return AArch64::SBCSXr;
1772 case AArch64::ANDWri:
1773 return AArch64::ANDSWri;
1774 case AArch64::ANDXri:
1775 return AArch64::ANDSXri;
1781 for (
auto *BB :
MBB->successors())
1782 if (BB->isLiveIn(AArch64::NZCV))
1791 switch (Instr.getOpcode()) {
1795 case AArch64::Bcc: {
1796 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1801 case AArch64::CSINVWr:
1802 case AArch64::CSINVXr:
1803 case AArch64::CSINCWr:
1804 case AArch64::CSINCXr:
1805 case AArch64::CSELWr:
1806 case AArch64::CSELXr:
1807 case AArch64::CSNEGWr:
1808 case AArch64::CSNEGXr:
1809 case AArch64::FCSELSrrr:
1810 case AArch64::FCSELDrrr: {
1811 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1824 Instr.getOperand(CCIdx).
getImm())
1877std::optional<UsedNZCV>
1882 if (
MI.getParent() != CmpParent)
1883 return std::nullopt;
1886 return std::nullopt;
1891 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1894 return std::nullopt;
1899 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1902 return NZCVUsedAfterCmp;
1906 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1910 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1932 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1938 "Caller guarantees that CmpInstr compares with constant 0");
1941 if (!NZVCUsed || NZVCUsed->C)
1963bool AArch64InstrInfo::substituteCmpToZero(
1967 MachineInstr *
MI =
MRI.getUniqueVRegDef(SrcReg);
1974 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1981 MI->setDesc(
get(NewOpc));
1986 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1998 assert((CmpValue == 0 || CmpValue == 1) &&
1999 "Only comparisons to 0 or 1 considered for removal!");
2002 unsigned MIOpc =
MI.getOpcode();
2003 if (MIOpc == AArch64::CSINCWr) {
2004 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
2005 MI.getOperand(2).getReg() != AArch64::WZR)
2007 }
else if (MIOpc == AArch64::CSINCXr) {
2008 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
2009 MI.getOperand(2).getReg() != AArch64::XZR)
2019 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
2023 const unsigned CmpOpcode = CmpInstr.
getOpcode();
2025 if (CmpValue && !IsSubsRegImm)
2027 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
2032 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
2035 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2039 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2042 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
2043 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
2046 if (MIUsedNZCV.
N && !CmpValue)
2088bool AArch64InstrInfo::removeCmpToZeroOrOne(
2091 MachineInstr *
MI =
MRI.getUniqueVRegDef(SrcReg);
2095 SmallVector<MachineInstr *, 4> CCUseInstrs;
2096 bool IsInvertCC =
false;
2104 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2106 assert(Idx >= 0 &&
"Unexpected instruction using CC.");
2107 MachineOperand &CCOperand = CCUseInstr->getOperand(Idx);
2116bool AArch64InstrInfo::expandPostRAPseudo(
MachineInstr &
MI)
const {
2117 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2118 MI.getOpcode() != AArch64::CATCHRET)
2121 MachineBasicBlock &
MBB = *
MI.getParent();
2123 auto TRI = Subtarget.getRegisterInfo();
2126 if (
MI.getOpcode() == AArch64::CATCHRET) {
2128 const TargetInstrInfo *
TII =
2130 MachineBasicBlock *TargetMBB =
MI.getOperand(0).getMBB();
2135 FirstEpilogSEH = std::prev(FirstEpilogSEH);
2137 FirstEpilogSEH = std::next(FirstEpilogSEH);
2152 if (
M.getStackProtectorGuard() ==
"sysreg") {
2153 const AArch64SysReg::SysReg *SrcReg =
2154 AArch64SysReg::lookupSysRegByName(
M.getStackProtectorGuardReg());
2162 int Offset =
M.getStackProtectorGuardOffset();
2213 const GlobalValue *GV =
2216 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2222 if (Subtarget.isTargetILP32()) {
2223 unsigned Reg32 =
TRI->getSubReg(
Reg, AArch64::sub_32);
2237 assert(!Subtarget.isTargetILP32() &&
"how can large exist in ILP32?");
2264 if (Subtarget.isTargetILP32()) {
2265 unsigned Reg32 =
TRI->getSubReg(
Reg, AArch64::sub_32);
2288 switch (
MI.getOpcode()) {
2291 case AArch64::MOVZWi:
2292 case AArch64::MOVZXi:
2293 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2294 assert(
MI.getDesc().getNumOperands() == 3 &&
2295 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2299 case AArch64::ANDWri:
2300 return MI.getOperand(1).getReg() == AArch64::WZR;
2301 case AArch64::ANDXri:
2302 return MI.getOperand(1).getReg() == AArch64::XZR;
2303 case TargetOpcode::COPY:
2304 return MI.getOperand(1).getReg() == AArch64::WZR;
2312 switch (
MI.getOpcode()) {
2315 case TargetOpcode::COPY: {
2318 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2319 AArch64::GPR64RegClass.
contains(DstReg));
2321 case AArch64::ORRXrs:
2322 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2323 assert(
MI.getDesc().getNumOperands() == 4 &&
2324 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2328 case AArch64::ADDXri:
2329 if (
MI.getOperand(2).getImm() == 0) {
2330 assert(
MI.getDesc().getNumOperands() == 4 &&
2331 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2342 switch (
MI.getOpcode()) {
2345 case TargetOpcode::COPY: {
2347 return AArch64::FPR128RegClass.contains(DstReg);
2349 case AArch64::ORRv16i8:
2350 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2351 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2352 "invalid ORRv16i8 operands");
2361 int &FrameIndex)
const {
2362 switch (
MI.getOpcode()) {
2365 case AArch64::LDRWui:
2366 case AArch64::LDRXui:
2367 case AArch64::LDRBui:
2368 case AArch64::LDRHui:
2369 case AArch64::LDRSui:
2370 case AArch64::LDRDui:
2371 case AArch64::LDRQui:
2372 case AArch64::LDR_PXI:
2373 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2374 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2375 FrameIndex =
MI.getOperand(1).getIndex();
2376 return MI.getOperand(0).getReg();
2385 int &FrameIndex)
const {
2386 switch (
MI.getOpcode()) {
2389 case AArch64::STRWui:
2390 case AArch64::STRXui:
2391 case AArch64::STRBui:
2392 case AArch64::STRHui:
2393 case AArch64::STRSui:
2394 case AArch64::STRDui:
2395 case AArch64::STRQui:
2396 case AArch64::STR_PXI:
2397 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2398 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2399 FrameIndex =
MI.getOperand(1).getIndex();
2400 return MI.getOperand(0).getReg();
2410 return MMO->getFlags() & MOSuppressPair;
2416 if (
MI.memoperands_empty())
2424 return MMO->getFlags() & MOStridedAccess;
2432 case AArch64::STURSi:
2433 case AArch64::STRSpre:
2434 case AArch64::STURDi:
2435 case AArch64::STRDpre:
2436 case AArch64::STURQi:
2437 case AArch64::STRQpre:
2438 case AArch64::STURBBi:
2439 case AArch64::STURHHi:
2440 case AArch64::STURWi:
2441 case AArch64::STRWpre:
2442 case AArch64::STURXi:
2443 case AArch64::STRXpre:
2444 case AArch64::LDURSi:
2445 case AArch64::LDRSpre:
2446 case AArch64::LDURDi:
2447 case AArch64::LDRDpre:
2448 case AArch64::LDURQi:
2449 case AArch64::LDRQpre:
2450 case AArch64::LDURWi:
2451 case AArch64::LDRWpre:
2452 case AArch64::LDURXi:
2453 case AArch64::LDRXpre:
2454 case AArch64::LDRSWpre:
2455 case AArch64::LDURSWi:
2456 case AArch64::LDURHHi:
2457 case AArch64::LDURBBi:
2458 case AArch64::LDURSBWi:
2459 case AArch64::LDURSHWi:
2467 case AArch64::PRFMui:
return AArch64::PRFUMi;
2468 case AArch64::LDRXui:
return AArch64::LDURXi;
2469 case AArch64::LDRWui:
return AArch64::LDURWi;
2470 case AArch64::LDRBui:
return AArch64::LDURBi;
2471 case AArch64::LDRHui:
return AArch64::LDURHi;
2472 case AArch64::LDRSui:
return AArch64::LDURSi;
2473 case AArch64::LDRDui:
return AArch64::LDURDi;
2474 case AArch64::LDRQui:
return AArch64::LDURQi;
2475 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2476 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2477 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2478 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2479 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2480 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2481 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2482 case AArch64::STRXui:
return AArch64::STURXi;
2483 case AArch64::STRWui:
return AArch64::STURWi;
2484 case AArch64::STRBui:
return AArch64::STURBi;
2485 case AArch64::STRHui:
return AArch64::STURHi;
2486 case AArch64::STRSui:
return AArch64::STURSi;
2487 case AArch64::STRDui:
return AArch64::STURDi;
2488 case AArch64::STRQui:
return AArch64::STURQi;
2489 case AArch64::STRBBui:
return AArch64::STURBBi;
2490 case AArch64::STRHHui:
return AArch64::STURHHi;
2499 case AArch64::LDAPURBi:
2500 case AArch64::LDAPURHi:
2501 case AArch64::LDAPURi:
2502 case AArch64::LDAPURSBWi:
2503 case AArch64::LDAPURSBXi:
2504 case AArch64::LDAPURSHWi:
2505 case AArch64::LDAPURSHXi:
2506 case AArch64::LDAPURSWi:
2507 case AArch64::LDAPURXi:
2508 case AArch64::LDR_PPXI:
2509 case AArch64::LDR_PXI:
2510 case AArch64::LDR_ZXI:
2511 case AArch64::LDR_ZZXI:
2512 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
2513 case AArch64::LDR_ZZZXI:
2514 case AArch64::LDR_ZZZZXI:
2515 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
2516 case AArch64::LDRBBui:
2517 case AArch64::LDRBui:
2518 case AArch64::LDRDui:
2519 case AArch64::LDRHHui:
2520 case AArch64::LDRHui:
2521 case AArch64::LDRQui:
2522 case AArch64::LDRSBWui:
2523 case AArch64::LDRSBXui:
2524 case AArch64::LDRSHWui:
2525 case AArch64::LDRSHXui:
2526 case AArch64::LDRSui:
2527 case AArch64::LDRSWui:
2528 case AArch64::LDRWui:
2529 case AArch64::LDRXui:
2530 case AArch64::LDURBBi:
2531 case AArch64::LDURBi:
2532 case AArch64::LDURDi:
2533 case AArch64::LDURHHi:
2534 case AArch64::LDURHi:
2535 case AArch64::LDURQi:
2536 case AArch64::LDURSBWi:
2537 case AArch64::LDURSBXi:
2538 case AArch64::LDURSHWi:
2539 case AArch64::LDURSHXi:
2540 case AArch64::LDURSi:
2541 case AArch64::LDURSWi:
2542 case AArch64::LDURWi:
2543 case AArch64::LDURXi:
2544 case AArch64::PRFMui:
2545 case AArch64::PRFUMi:
2546 case AArch64::ST2Gi:
2548 case AArch64::STLURBi:
2549 case AArch64::STLURHi:
2550 case AArch64::STLURWi:
2551 case AArch64::STLURXi:
2552 case AArch64::StoreSwiftAsyncContext:
2553 case AArch64::STR_PPXI:
2554 case AArch64::STR_PXI:
2555 case AArch64::STR_ZXI:
2556 case AArch64::STR_ZZXI:
2557 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
2558 case AArch64::STR_ZZZXI:
2559 case AArch64::STR_ZZZZXI:
2560 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
2561 case AArch64::STRBBui:
2562 case AArch64::STRBui:
2563 case AArch64::STRDui:
2564 case AArch64::STRHHui:
2565 case AArch64::STRHui:
2566 case AArch64::STRQui:
2567 case AArch64::STRSui:
2568 case AArch64::STRWui:
2569 case AArch64::STRXui:
2570 case AArch64::STURBBi:
2571 case AArch64::STURBi:
2572 case AArch64::STURDi:
2573 case AArch64::STURHHi:
2574 case AArch64::STURHi:
2575 case AArch64::STURQi:
2576 case AArch64::STURSi:
2577 case AArch64::STURWi:
2578 case AArch64::STURXi:
2579 case AArch64::STZ2Gi:
2580 case AArch64::STZGi:
2581 case AArch64::TAGPstack:
2583 case AArch64::LD1B_D_IMM:
2584 case AArch64::LD1B_H_IMM:
2585 case AArch64::LD1B_IMM:
2586 case AArch64::LD1B_S_IMM:
2587 case AArch64::LD1D_IMM:
2588 case AArch64::LD1H_D_IMM:
2589 case AArch64::LD1H_IMM:
2590 case AArch64::LD1H_S_IMM:
2591 case AArch64::LD1RB_D_IMM:
2592 case AArch64::LD1RB_H_IMM:
2593 case AArch64::LD1RB_IMM:
2594 case AArch64::LD1RB_S_IMM:
2595 case AArch64::LD1RD_IMM:
2596 case AArch64::LD1RH_D_IMM:
2597 case AArch64::LD1RH_IMM:
2598 case AArch64::LD1RH_S_IMM:
2599 case AArch64::LD1RSB_D_IMM:
2600 case AArch64::LD1RSB_H_IMM:
2601 case AArch64::LD1RSB_S_IMM:
2602 case AArch64::LD1RSH_D_IMM:
2603 case AArch64::LD1RSH_S_IMM:
2604 case AArch64::LD1RSW_IMM:
2605 case AArch64::LD1RW_D_IMM:
2606 case AArch64::LD1RW_IMM:
2607 case AArch64::LD1SB_D_IMM:
2608 case AArch64::LD1SB_H_IMM:
2609 case AArch64::LD1SB_S_IMM:
2610 case AArch64::LD1SH_D_IMM:
2611 case AArch64::LD1SH_S_IMM:
2612 case AArch64::LD1SW_D_IMM:
2613 case AArch64::LD1W_D_IMM:
2614 case AArch64::LD1W_IMM:
2615 case AArch64::LD2B_IMM:
2616 case AArch64::LD2D_IMM:
2617 case AArch64::LD2H_IMM:
2618 case AArch64::LD2W_IMM:
2619 case AArch64::LD3B_IMM:
2620 case AArch64::LD3D_IMM:
2621 case AArch64::LD3H_IMM:
2622 case AArch64::LD3W_IMM:
2623 case AArch64::LD4B_IMM:
2624 case AArch64::LD4D_IMM:
2625 case AArch64::LD4H_IMM:
2626 case AArch64::LD4W_IMM:
2628 case AArch64::LDNF1B_D_IMM:
2629 case AArch64::LDNF1B_H_IMM:
2630 case AArch64::LDNF1B_IMM:
2631 case AArch64::LDNF1B_S_IMM:
2632 case AArch64::LDNF1D_IMM:
2633 case AArch64::LDNF1H_D_IMM:
2634 case AArch64::LDNF1H_IMM:
2635 case AArch64::LDNF1H_S_IMM:
2636 case AArch64::LDNF1SB_D_IMM:
2637 case AArch64::LDNF1SB_H_IMM:
2638 case AArch64::LDNF1SB_S_IMM:
2639 case AArch64::LDNF1SH_D_IMM:
2640 case AArch64::LDNF1SH_S_IMM:
2641 case AArch64::LDNF1SW_D_IMM:
2642 case AArch64::LDNF1W_D_IMM:
2643 case AArch64::LDNF1W_IMM:
2644 case AArch64::LDNPDi:
2645 case AArch64::LDNPQi:
2646 case AArch64::LDNPSi:
2647 case AArch64::LDNPWi:
2648 case AArch64::LDNPXi:
2649 case AArch64::LDNT1B_ZRI:
2650 case AArch64::LDNT1D_ZRI:
2651 case AArch64::LDNT1H_ZRI:
2652 case AArch64::LDNT1W_ZRI:
2653 case AArch64::LDPDi:
2654 case AArch64::LDPQi:
2655 case AArch64::LDPSi:
2656 case AArch64::LDPWi:
2657 case AArch64::LDPXi:
2658 case AArch64::LDRBBpost:
2659 case AArch64::LDRBBpre:
2660 case AArch64::LDRBpost:
2661 case AArch64::LDRBpre:
2662 case AArch64::LDRDpost:
2663 case AArch64::LDRDpre:
2664 case AArch64::LDRHHpost:
2665 case AArch64::LDRHHpre:
2666 case AArch64::LDRHpost:
2667 case AArch64::LDRHpre:
2668 case AArch64::LDRQpost:
2669 case AArch64::LDRQpre:
2670 case AArch64::LDRSpost:
2671 case AArch64::LDRSpre:
2672 case AArch64::LDRWpost:
2673 case AArch64::LDRWpre:
2674 case AArch64::LDRXpost:
2675 case AArch64::LDRXpre:
2676 case AArch64::ST1B_D_IMM:
2677 case AArch64::ST1B_H_IMM:
2678 case AArch64::ST1B_IMM:
2679 case AArch64::ST1B_S_IMM:
2680 case AArch64::ST1D_IMM:
2681 case AArch64::ST1H_D_IMM:
2682 case AArch64::ST1H_IMM:
2683 case AArch64::ST1H_S_IMM:
2684 case AArch64::ST1W_D_IMM:
2685 case AArch64::ST1W_IMM:
2686 case AArch64::ST2B_IMM:
2687 case AArch64::ST2D_IMM:
2688 case AArch64::ST2H_IMM:
2689 case AArch64::ST2W_IMM:
2690 case AArch64::ST3B_IMM:
2691 case AArch64::ST3D_IMM:
2692 case AArch64::ST3H_IMM:
2693 case AArch64::ST3W_IMM:
2694 case AArch64::ST4B_IMM:
2695 case AArch64::ST4D_IMM:
2696 case AArch64::ST4H_IMM:
2697 case AArch64::ST4W_IMM:
2698 case AArch64::STGPi:
2699 case AArch64::STGPreIndex:
2700 case AArch64::STZGPreIndex:
2701 case AArch64::ST2GPreIndex:
2702 case AArch64::STZ2GPreIndex:
2703 case AArch64::STGPostIndex:
2704 case AArch64::STZGPostIndex:
2705 case AArch64::ST2GPostIndex:
2706 case AArch64::STZ2GPostIndex:
2707 case AArch64::STNPDi:
2708 case AArch64::STNPQi:
2709 case AArch64::STNPSi:
2710 case AArch64::STNPWi:
2711 case AArch64::STNPXi:
2712 case AArch64::STNT1B_ZRI:
2713 case AArch64::STNT1D_ZRI:
2714 case AArch64::STNT1H_ZRI:
2715 case AArch64::STNT1W_ZRI:
2716 case AArch64::STPDi:
2717 case AArch64::STPQi:
2718 case AArch64::STPSi:
2719 case AArch64::STPWi:
2720 case AArch64::STPXi:
2721 case AArch64::STRBBpost:
2722 case AArch64::STRBBpre:
2723 case AArch64::STRBpost:
2724 case AArch64::STRBpre:
2725 case AArch64::STRDpost:
2726 case AArch64::STRDpre:
2727 case AArch64::STRHHpost:
2728 case AArch64::STRHHpre:
2729 case AArch64::STRHpost:
2730 case AArch64::STRHpre:
2731 case AArch64::STRQpost:
2732 case AArch64::STRQpre:
2733 case AArch64::STRSpost:
2734 case AArch64::STRSpre:
2735 case AArch64::STRWpost:
2736 case AArch64::STRWpre:
2737 case AArch64::STRXpost:
2738 case AArch64::STRXpre:
2740 case AArch64::LDPDpost:
2741 case AArch64::LDPDpre:
2742 case AArch64::LDPQpost:
2743 case AArch64::LDPQpre:
2744 case AArch64::LDPSpost:
2745 case AArch64::LDPSpre:
2746 case AArch64::LDPWpost:
2747 case AArch64::LDPWpre:
2748 case AArch64::LDPXpost:
2749 case AArch64::LDPXpre:
2750 case AArch64::STGPpre:
2751 case AArch64::STGPpost:
2752 case AArch64::STPDpost:
2753 case AArch64::STPDpre:
2754 case AArch64::STPQpost:
2755 case AArch64::STPQpre:
2756 case AArch64::STPSpost:
2757 case AArch64::STPSpre:
2758 case AArch64::STPWpost:
2759 case AArch64::STPWpre:
2760 case AArch64::STPXpost:
2761 case AArch64::STPXpre:
2767 switch (
MI.getOpcode()) {
2771 case AArch64::STRSui:
2772 case AArch64::STRDui:
2773 case AArch64::STRQui:
2774 case AArch64::STRXui:
2775 case AArch64::STRWui:
2776 case AArch64::LDRSui:
2777 case AArch64::LDRDui:
2778 case AArch64::LDRQui:
2779 case AArch64::LDRXui:
2780 case AArch64::LDRWui:
2781 case AArch64::LDRSWui:
2783 case AArch64::STURSi:
2784 case AArch64::STRSpre:
2785 case AArch64::STURDi:
2786 case AArch64::STRDpre:
2787 case AArch64::STURQi:
2788 case AArch64::STRQpre:
2789 case AArch64::STURWi:
2790 case AArch64::STRWpre:
2791 case AArch64::STURXi:
2792 case AArch64::STRXpre:
2793 case AArch64::LDURSi:
2794 case AArch64::LDRSpre:
2795 case AArch64::LDURDi:
2796 case AArch64::LDRDpre:
2797 case AArch64::LDURQi:
2798 case AArch64::LDRQpre:
2799 case AArch64::LDURWi:
2800 case AArch64::LDRWpre:
2801 case AArch64::LDURXi:
2802 case AArch64::LDRXpre:
2803 case AArch64::LDURSWi:
2804 case AArch64::LDRSWpre:
2806 case AArch64::LDR_ZXI:
2807 case AArch64::STR_ZXI:
2813 switch (
MI.getOpcode()) {
2816 "Unexpected instruction - was a new tail call opcode introduced?");
2818 case AArch64::TCRETURNdi:
2819 case AArch64::TCRETURNri:
2820 case AArch64::TCRETURNrix16x17:
2821 case AArch64::TCRETURNrix17:
2822 case AArch64::TCRETURNrinotx16:
2823 case AArch64::TCRETURNriALL:
2824 case AArch64::AUTH_TCRETURN:
2825 case AArch64::AUTH_TCRETURN_BTI:
2835 case AArch64::ADDWri:
2836 return AArch64::ADDSWri;
2837 case AArch64::ADDWrr:
2838 return AArch64::ADDSWrr;
2839 case AArch64::ADDWrs:
2840 return AArch64::ADDSWrs;
2841 case AArch64::ADDWrx:
2842 return AArch64::ADDSWrx;
2843 case AArch64::ANDWri:
2844 return AArch64::ANDSWri;
2845 case AArch64::ANDWrr:
2846 return AArch64::ANDSWrr;
2847 case AArch64::ANDWrs:
2848 return AArch64::ANDSWrs;
2849 case AArch64::BICWrr:
2850 return AArch64::BICSWrr;
2851 case AArch64::BICWrs:
2852 return AArch64::BICSWrs;
2853 case AArch64::SUBWri:
2854 return AArch64::SUBSWri;
2855 case AArch64::SUBWrr:
2856 return AArch64::SUBSWrr;
2857 case AArch64::SUBWrs:
2858 return AArch64::SUBSWrs;
2859 case AArch64::SUBWrx:
2860 return AArch64::SUBSWrx;
2862 case AArch64::ADDXri:
2863 return AArch64::ADDSXri;
2864 case AArch64::ADDXrr:
2865 return AArch64::ADDSXrr;
2866 case AArch64::ADDXrs:
2867 return AArch64::ADDSXrs;
2868 case AArch64::ADDXrx:
2869 return AArch64::ADDSXrx;
2870 case AArch64::ANDXri:
2871 return AArch64::ANDSXri;
2872 case AArch64::ANDXrr:
2873 return AArch64::ANDSXrr;
2874 case AArch64::ANDXrs:
2875 return AArch64::ANDSXrs;
2876 case AArch64::BICXrr:
2877 return AArch64::BICSXrr;
2878 case AArch64::BICXrs:
2879 return AArch64::BICSXrs;
2880 case AArch64::SUBXri:
2881 return AArch64::SUBSXri;
2882 case AArch64::SUBXrr:
2883 return AArch64::SUBSXrr;
2884 case AArch64::SUBXrs:
2885 return AArch64::SUBSXrs;
2886 case AArch64::SUBXrx:
2887 return AArch64::SUBSXrx;
2889 case AArch64::AND_PPzPP:
2890 return AArch64::ANDS_PPzPP;
2891 case AArch64::BIC_PPzPP:
2892 return AArch64::BICS_PPzPP;
2893 case AArch64::EOR_PPzPP:
2894 return AArch64::EORS_PPzPP;
2895 case AArch64::NAND_PPzPP:
2896 return AArch64::NANDS_PPzPP;
2897 case AArch64::NOR_PPzPP:
2898 return AArch64::NORS_PPzPP;
2899 case AArch64::ORN_PPzPP:
2900 return AArch64::ORNS_PPzPP;
2901 case AArch64::ORR_PPzPP:
2902 return AArch64::ORRS_PPzPP;
2903 case AArch64::BRKA_PPzP:
2904 return AArch64::BRKAS_PPzP;
2905 case AArch64::BRKPA_PPzPP:
2906 return AArch64::BRKPAS_PPzPP;
2907 case AArch64::BRKB_PPzP:
2908 return AArch64::BRKBS_PPzP;
2909 case AArch64::BRKPB_PPzPP:
2910 return AArch64::BRKPBS_PPzPP;
2911 case AArch64::BRKN_PPzP:
2912 return AArch64::BRKNS_PPzP;
2913 case AArch64::RDFFR_PPz:
2914 return AArch64::RDFFRS_PPz;
2915 case AArch64::PTRUE_B:
2916 return AArch64::PTRUES_B;
2927 if (
MI.hasOrderedMemoryRef())
2932 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2933 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2934 "Expected a reg or frame index operand.");
2938 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2940 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2953 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2954 Register BaseReg =
MI.getOperand(1).getReg();
2956 if (
MI.modifiesRegister(BaseReg,
TRI))
2962 switch (
MI.getOpcode()) {
2965 case AArch64::LDR_ZXI:
2966 case AArch64::STR_ZXI:
2967 if (!Subtarget.isLittleEndian() ||
2968 Subtarget.getSVEVectorSizeInBits() != 128)
2981 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2983 MI.getMF()->getFunction().needsUnwindTableEntry();
2989 if (Subtarget.isPaired128Slow()) {
2990 switch (
MI.getOpcode()) {
2993 case AArch64::LDURQi:
2994 case AArch64::STURQi:
2995 case AArch64::LDRQui:
2996 case AArch64::STRQui:
3023std::optional<ExtAddrMode>
3028 bool OffsetIsScalable;
3029 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
3030 return std::nullopt;
3033 return std::nullopt;
3048 int64_t OffsetScale = 1;
3053 case AArch64::LDURQi:
3054 case AArch64::STURQi:
3058 case AArch64::LDURDi:
3059 case AArch64::STURDi:
3060 case AArch64::LDURXi:
3061 case AArch64::STURXi:
3065 case AArch64::LDURWi:
3066 case AArch64::LDURSWi:
3067 case AArch64::STURWi:
3071 case AArch64::LDURHi:
3072 case AArch64::STURHi:
3073 case AArch64::LDURHHi:
3074 case AArch64::STURHHi:
3075 case AArch64::LDURSHXi:
3076 case AArch64::LDURSHWi:
3080 case AArch64::LDRBroX:
3081 case AArch64::LDRBBroX:
3082 case AArch64::LDRSBXroX:
3083 case AArch64::LDRSBWroX:
3084 case AArch64::STRBroX:
3085 case AArch64::STRBBroX:
3086 case AArch64::LDURBi:
3087 case AArch64::LDURBBi:
3088 case AArch64::LDURSBXi:
3089 case AArch64::LDURSBWi:
3090 case AArch64::STURBi:
3091 case AArch64::STURBBi:
3092 case AArch64::LDRBui:
3093 case AArch64::LDRBBui:
3094 case AArch64::LDRSBXui:
3095 case AArch64::LDRSBWui:
3096 case AArch64::STRBui:
3097 case AArch64::STRBBui:
3101 case AArch64::LDRQroX:
3102 case AArch64::STRQroX:
3103 case AArch64::LDRQui:
3104 case AArch64::STRQui:
3109 case AArch64::LDRDroX:
3110 case AArch64::STRDroX:
3111 case AArch64::LDRXroX:
3112 case AArch64::STRXroX:
3113 case AArch64::LDRDui:
3114 case AArch64::STRDui:
3115 case AArch64::LDRXui:
3116 case AArch64::STRXui:
3121 case AArch64::LDRWroX:
3122 case AArch64::LDRSWroX:
3123 case AArch64::STRWroX:
3124 case AArch64::LDRWui:
3125 case AArch64::LDRSWui:
3126 case AArch64::STRWui:
3131 case AArch64::LDRHroX:
3132 case AArch64::STRHroX:
3133 case AArch64::LDRHHroX:
3134 case AArch64::STRHHroX:
3135 case AArch64::LDRSHXroX:
3136 case AArch64::LDRSHWroX:
3137 case AArch64::LDRHui:
3138 case AArch64::STRHui:
3139 case AArch64::LDRHHui:
3140 case AArch64::STRHHui:
3141 case AArch64::LDRSHXui:
3142 case AArch64::LDRSHWui:
3150 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
3174 case AArch64::SBFMXri:
3187 AM.
Scale = OffsetScale;
3192 case TargetOpcode::SUBREG_TO_REG: {
3205 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3209 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3210 DefMI.getOperand(1).getReg() != AArch64::WZR ||
3211 DefMI.getOperand(3).getImm() != 0)
3218 AM.
Scale = OffsetScale;
3229 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3230 int64_t NewOffset) ->
bool {
3231 int64_t MinOffset, MaxOffset;
3248 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3249 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3251 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3253 int64_t NewOffset = OldOffset + Disp;
3254 if (!isLegalAddressingMode(NumBytes, NewOffset, 0))
3258 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3268 auto canFoldAddRegIntoAddrMode =
3273 if ((
unsigned)Scale != Scale)
3275 if (!isLegalAddressingMode(NumBytes, 0, Scale))
3287 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3288 Subtarget.isSTRQroSlow();
3297 case AArch64::ADDXri:
3303 return canFoldAddSubImmIntoAddrMode(Disp);
3305 case AArch64::SUBXri:
3311 return canFoldAddSubImmIntoAddrMode(-Disp);
3313 case AArch64::ADDXrs: {
3326 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3328 if (avoidSlowSTRQ(MemI))
3331 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3334 case AArch64::ADDXrr:
3342 if (!OptSize && avoidSlowSTRQ(MemI))
3344 return canFoldAddRegIntoAddrMode(1);
3346 case AArch64::ADDXrx:
3354 if (!OptSize && avoidSlowSTRQ(MemI))
3363 return canFoldAddRegIntoAddrMode(
3378 case AArch64::LDURQi:
3379 case AArch64::LDRQui:
3380 return AArch64::LDRQroX;
3381 case AArch64::STURQi:
3382 case AArch64::STRQui:
3383 return AArch64::STRQroX;
3384 case AArch64::LDURDi:
3385 case AArch64::LDRDui:
3386 return AArch64::LDRDroX;
3387 case AArch64::STURDi:
3388 case AArch64::STRDui:
3389 return AArch64::STRDroX;
3390 case AArch64::LDURXi:
3391 case AArch64::LDRXui:
3392 return AArch64::LDRXroX;
3393 case AArch64::STURXi:
3394 case AArch64::STRXui:
3395 return AArch64::STRXroX;
3396 case AArch64::LDURWi:
3397 case AArch64::LDRWui:
3398 return AArch64::LDRWroX;
3399 case AArch64::LDURSWi:
3400 case AArch64::LDRSWui:
3401 return AArch64::LDRSWroX;
3402 case AArch64::STURWi:
3403 case AArch64::STRWui:
3404 return AArch64::STRWroX;
3405 case AArch64::LDURHi:
3406 case AArch64::LDRHui:
3407 return AArch64::LDRHroX;
3408 case AArch64::STURHi:
3409 case AArch64::STRHui:
3410 return AArch64::STRHroX;
3411 case AArch64::LDURHHi:
3412 case AArch64::LDRHHui:
3413 return AArch64::LDRHHroX;
3414 case AArch64::STURHHi:
3415 case AArch64::STRHHui:
3416 return AArch64::STRHHroX;
3417 case AArch64::LDURSHXi:
3418 case AArch64::LDRSHXui:
3419 return AArch64::LDRSHXroX;
3420 case AArch64::LDURSHWi:
3421 case AArch64::LDRSHWui:
3422 return AArch64::LDRSHWroX;
3423 case AArch64::LDURBi:
3424 case AArch64::LDRBui:
3425 return AArch64::LDRBroX;
3426 case AArch64::LDURBBi:
3427 case AArch64::LDRBBui:
3428 return AArch64::LDRBBroX;
3429 case AArch64::LDURSBXi:
3430 case AArch64::LDRSBXui:
3431 return AArch64::LDRSBXroX;
3432 case AArch64::LDURSBWi:
3433 case AArch64::LDRSBWui:
3434 return AArch64::LDRSBWroX;
3435 case AArch64::STURBi:
3436 case AArch64::STRBui:
3437 return AArch64::STRBroX;
3438 case AArch64::STURBBi:
3439 case AArch64::STRBBui:
3440 return AArch64::STRBBroX;
3452 case AArch64::LDURQi:
3454 return AArch64::LDRQui;
3455 case AArch64::STURQi:
3457 return AArch64::STRQui;
3458 case AArch64::LDURDi:
3460 return AArch64::LDRDui;
3461 case AArch64::STURDi:
3463 return AArch64::STRDui;
3464 case AArch64::LDURXi:
3466 return AArch64::LDRXui;
3467 case AArch64::STURXi:
3469 return AArch64::STRXui;
3470 case AArch64::LDURWi:
3472 return AArch64::LDRWui;
3473 case AArch64::LDURSWi:
3475 return AArch64::LDRSWui;
3476 case AArch64::STURWi:
3478 return AArch64::STRWui;
3479 case AArch64::LDURHi:
3481 return AArch64::LDRHui;
3482 case AArch64::STURHi:
3484 return AArch64::STRHui;
3485 case AArch64::LDURHHi:
3487 return AArch64::LDRHHui;
3488 case AArch64::STURHHi:
3490 return AArch64::STRHHui;
3491 case AArch64::LDURSHXi:
3493 return AArch64::LDRSHXui;
3494 case AArch64::LDURSHWi:
3496 return AArch64::LDRSHWui;
3497 case AArch64::LDURBi:
3499 return AArch64::LDRBui;
3500 case AArch64::LDURBBi:
3502 return AArch64::LDRBBui;
3503 case AArch64::LDURSBXi:
3505 return AArch64::LDRSBXui;
3506 case AArch64::LDURSBWi:
3508 return AArch64::LDRSBWui;
3509 case AArch64::STURBi:
3511 return AArch64::STRBui;
3512 case AArch64::STURBBi:
3514 return AArch64::STRBBui;
3515 case AArch64::LDRQui:
3516 case AArch64::STRQui:
3519 case AArch64::LDRDui:
3520 case AArch64::STRDui:
3521 case AArch64::LDRXui:
3522 case AArch64::STRXui:
3525 case AArch64::LDRWui:
3526 case AArch64::LDRSWui:
3527 case AArch64::STRWui:
3530 case AArch64::LDRHui:
3531 case AArch64::STRHui:
3532 case AArch64::LDRHHui:
3533 case AArch64::STRHHui:
3534 case AArch64::LDRSHXui:
3535 case AArch64::LDRSHWui:
3538 case AArch64::LDRBui:
3539 case AArch64::LDRBBui:
3540 case AArch64::LDRSBXui:
3541 case AArch64::LDRSBWui:
3542 case AArch64::STRBui:
3543 case AArch64::STRBBui:
3557 case AArch64::LDURQi:
3558 case AArch64::STURQi:
3559 case AArch64::LDURDi:
3560 case AArch64::STURDi:
3561 case AArch64::LDURXi:
3562 case AArch64::STURXi:
3563 case AArch64::LDURWi:
3564 case AArch64::LDURSWi:
3565 case AArch64::STURWi:
3566 case AArch64::LDURHi:
3567 case AArch64::STURHi:
3568 case AArch64::LDURHHi:
3569 case AArch64::STURHHi:
3570 case AArch64::LDURSHXi:
3571 case AArch64::LDURSHWi:
3572 case AArch64::LDURBi:
3573 case AArch64::STURBi:
3574 case AArch64::LDURBBi:
3575 case AArch64::STURBBi:
3576 case AArch64::LDURSBWi:
3577 case AArch64::LDURSBXi:
3579 case AArch64::LDRQui:
3580 return AArch64::LDURQi;
3581 case AArch64::STRQui:
3582 return AArch64::STURQi;
3583 case AArch64::LDRDui:
3584 return AArch64::LDURDi;
3585 case AArch64::STRDui:
3586 return AArch64::STURDi;
3587 case AArch64::LDRXui:
3588 return AArch64::LDURXi;
3589 case AArch64::STRXui:
3590 return AArch64::STURXi;
3591 case AArch64::LDRWui:
3592 return AArch64::LDURWi;
3593 case AArch64::LDRSWui:
3594 return AArch64::LDURSWi;
3595 case AArch64::STRWui:
3596 return AArch64::STURWi;
3597 case AArch64::LDRHui:
3598 return AArch64::LDURHi;
3599 case AArch64::STRHui:
3600 return AArch64::STURHi;
3601 case AArch64::LDRHHui:
3602 return AArch64::LDURHHi;
3603 case AArch64::STRHHui:
3604 return AArch64::STURHHi;
3605 case AArch64::LDRSHXui:
3606 return AArch64::LDURSHXi;
3607 case AArch64::LDRSHWui:
3608 return AArch64::LDURSHWi;
3609 case AArch64::LDRBBui:
3610 return AArch64::LDURBBi;
3611 case AArch64::LDRBui:
3612 return AArch64::LDURBi;
3613 case AArch64::STRBBui:
3614 return AArch64::STURBBi;
3615 case AArch64::STRBui:
3616 return AArch64::STURBi;
3617 case AArch64::LDRSBWui:
3618 return AArch64::LDURSBWi;
3619 case AArch64::LDRSBXui:
3620 return AArch64::LDURSBXi;
3633 case AArch64::LDRQroX:
3634 case AArch64::LDURQi:
3635 case AArch64::LDRQui:
3636 return AArch64::LDRQroW;
3637 case AArch64::STRQroX:
3638 case AArch64::STURQi:
3639 case AArch64::STRQui:
3640 return AArch64::STRQroW;
3641 case AArch64::LDRDroX:
3642 case AArch64::LDURDi:
3643 case AArch64::LDRDui:
3644 return AArch64::LDRDroW;
3645 case AArch64::STRDroX:
3646 case AArch64::STURDi:
3647 case AArch64::STRDui:
3648 return AArch64::STRDroW;
3649 case AArch64::LDRXroX:
3650 case AArch64::LDURXi:
3651 case AArch64::LDRXui:
3652 return AArch64::LDRXroW;
3653 case AArch64::STRXroX:
3654 case AArch64::STURXi:
3655 case AArch64::STRXui:
3656 return AArch64::STRXroW;
3657 case AArch64::LDRWroX:
3658 case AArch64::LDURWi:
3659 case AArch64::LDRWui:
3660 return AArch64::LDRWroW;
3661 case AArch64::LDRSWroX:
3662 case AArch64::LDURSWi:
3663 case AArch64::LDRSWui:
3664 return AArch64::LDRSWroW;
3665 case AArch64::STRWroX:
3666 case AArch64::STURWi:
3667 case AArch64::STRWui:
3668 return AArch64::STRWroW;
3669 case AArch64::LDRHroX:
3670 case AArch64::LDURHi:
3671 case AArch64::LDRHui:
3672 return AArch64::LDRHroW;
3673 case AArch64::STRHroX:
3674 case AArch64::STURHi:
3675 case AArch64::STRHui:
3676 return AArch64::STRHroW;
3677 case AArch64::LDRHHroX:
3678 case AArch64::LDURHHi:
3679 case AArch64::LDRHHui:
3680 return AArch64::LDRHHroW;
3681 case AArch64::STRHHroX:
3682 case AArch64::STURHHi:
3683 case AArch64::STRHHui:
3684 return AArch64::STRHHroW;
3685 case AArch64::LDRSHXroX:
3686 case AArch64::LDURSHXi:
3687 case AArch64::LDRSHXui:
3688 return AArch64::LDRSHXroW;
3689 case AArch64::LDRSHWroX:
3690 case AArch64::LDURSHWi:
3691 case AArch64::LDRSHWui:
3692 return AArch64::LDRSHWroW;
3693 case AArch64::LDRBroX:
3694 case AArch64::LDURBi:
3695 case AArch64::LDRBui:
3696 return AArch64::LDRBroW;
3697 case AArch64::LDRBBroX:
3698 case AArch64::LDURBBi:
3699 case AArch64::LDRBBui:
3700 return AArch64::LDRBBroW;
3701 case AArch64::LDRSBXroX:
3702 case AArch64::LDURSBXi:
3703 case AArch64::LDRSBXui:
3704 return AArch64::LDRSBXroW;
3705 case AArch64::LDRSBWroX:
3706 case AArch64::LDURSBWi:
3707 case AArch64::LDRSBWui:
3708 return AArch64::LDRSBWroW;
3709 case AArch64::STRBroX:
3710 case AArch64::STURBi:
3711 case AArch64::STRBui:
3712 return AArch64::STRBroW;
3713 case AArch64::STRBBroX:
3714 case AArch64::STURBBi:
3715 case AArch64::STRBBui:
3716 return AArch64::STRBBroW;
3731 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3741 return B.getInstr();
3745 "Addressing mode not supported for folding");
3762 return B.getInstr();
3769 "Address offset can be a register or an immediate, but not both");
3771 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3776 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3790 return B.getInstr();
3794 "Function must not be called with an addressing mode it can't handle");
3803 case AArch64::LD1Fourv16b_POST:
3804 case AArch64::LD1Fourv1d_POST:
3805 case AArch64::LD1Fourv2d_POST:
3806 case AArch64::LD1Fourv2s_POST:
3807 case AArch64::LD1Fourv4h_POST:
3808 case AArch64::LD1Fourv4s_POST:
3809 case AArch64::LD1Fourv8b_POST:
3810 case AArch64::LD1Fourv8h_POST:
3811 case AArch64::LD1Onev16b_POST:
3812 case AArch64::LD1Onev1d_POST:
3813 case AArch64::LD1Onev2d_POST:
3814 case AArch64::LD1Onev2s_POST:
3815 case AArch64::LD1Onev4h_POST:
3816 case AArch64::LD1Onev4s_POST:
3817 case AArch64::LD1Onev8b_POST:
3818 case AArch64::LD1Onev8h_POST:
3819 case AArch64::LD1Rv16b_POST:
3820 case AArch64::LD1Rv1d_POST:
3821 case AArch64::LD1Rv2d_POST:
3822 case AArch64::LD1Rv2s_POST:
3823 case AArch64::LD1Rv4h_POST:
3824 case AArch64::LD1Rv4s_POST:
3825 case AArch64::LD1Rv8b_POST:
3826 case AArch64::LD1Rv8h_POST:
3827 case AArch64::LD1Threev16b_POST:
3828 case AArch64::LD1Threev1d_POST:
3829 case AArch64::LD1Threev2d_POST:
3830 case AArch64::LD1Threev2s_POST:
3831 case AArch64::LD1Threev4h_POST:
3832 case AArch64::LD1Threev4s_POST:
3833 case AArch64::LD1Threev8b_POST:
3834 case AArch64::LD1Threev8h_POST:
3835 case AArch64::LD1Twov16b_POST:
3836 case AArch64::LD1Twov1d_POST:
3837 case AArch64::LD1Twov2d_POST:
3838 case AArch64::LD1Twov2s_POST:
3839 case AArch64::LD1Twov4h_POST:
3840 case AArch64::LD1Twov4s_POST:
3841 case AArch64::LD1Twov8b_POST:
3842 case AArch64::LD1Twov8h_POST:
3843 case AArch64::LD1i16_POST:
3844 case AArch64::LD1i32_POST:
3845 case AArch64::LD1i64_POST:
3846 case AArch64::LD1i8_POST:
3847 case AArch64::LD2Rv16b_POST:
3848 case AArch64::LD2Rv1d_POST:
3849 case AArch64::LD2Rv2d_POST:
3850 case AArch64::LD2Rv2s_POST:
3851 case AArch64::LD2Rv4h_POST:
3852 case AArch64::LD2Rv4s_POST:
3853 case AArch64::LD2Rv8b_POST:
3854 case AArch64::LD2Rv8h_POST:
3855 case AArch64::LD2Twov16b_POST:
3856 case AArch64::LD2Twov2d_POST:
3857 case AArch64::LD2Twov2s_POST:
3858 case AArch64::LD2Twov4h_POST:
3859 case AArch64::LD2Twov4s_POST:
3860 case AArch64::LD2Twov8b_POST:
3861 case AArch64::LD2Twov8h_POST:
3862 case AArch64::LD2i16_POST:
3863 case AArch64::LD2i32_POST:
3864 case AArch64::LD2i64_POST:
3865 case AArch64::LD2i8_POST:
3866 case AArch64::LD3Rv16b_POST:
3867 case AArch64::LD3Rv1d_POST:
3868 case AArch64::LD3Rv2d_POST:
3869 case AArch64::LD3Rv2s_POST:
3870 case AArch64::LD3Rv4h_POST:
3871 case AArch64::LD3Rv4s_POST:
3872 case AArch64::LD3Rv8b_POST:
3873 case AArch64::LD3Rv8h_POST:
3874 case AArch64::LD3Threev16b_POST:
3875 case AArch64::LD3Threev2d_POST:
3876 case AArch64::LD3Threev2s_POST:
3877 case AArch64::LD3Threev4h_POST:
3878 case AArch64::LD3Threev4s_POST:
3879 case AArch64::LD3Threev8b_POST:
3880 case AArch64::LD3Threev8h_POST:
3881 case AArch64::LD3i16_POST:
3882 case AArch64::LD3i32_POST:
3883 case AArch64::LD3i64_POST:
3884 case AArch64::LD3i8_POST:
3885 case AArch64::LD4Fourv16b_POST:
3886 case AArch64::LD4Fourv2d_POST:
3887 case AArch64::LD4Fourv2s_POST:
3888 case AArch64::LD4Fourv4h_POST:
3889 case AArch64::LD4Fourv4s_POST:
3890 case AArch64::LD4Fourv8b_POST:
3891 case AArch64::LD4Fourv8h_POST:
3892 case AArch64::LD4Rv16b_POST:
3893 case AArch64::LD4Rv1d_POST:
3894 case AArch64::LD4Rv2d_POST:
3895 case AArch64::LD4Rv2s_POST:
3896 case AArch64::LD4Rv4h_POST:
3897 case AArch64::LD4Rv4s_POST:
3898 case AArch64::LD4Rv8b_POST:
3899 case AArch64::LD4Rv8h_POST:
3900 case AArch64::LD4i16_POST:
3901 case AArch64::LD4i32_POST:
3902 case AArch64::LD4i64_POST:
3903 case AArch64::LD4i8_POST:
3904 case AArch64::LDAPRWpost:
3905 case AArch64::LDAPRXpost:
3906 case AArch64::LDIAPPWpost:
3907 case AArch64::LDIAPPXpost:
3908 case AArch64::LDPDpost:
3909 case AArch64::LDPQpost:
3910 case AArch64::LDPSWpost:
3911 case AArch64::LDPSpost:
3912 case AArch64::LDPWpost:
3913 case AArch64::LDPXpost:
3914 case AArch64::LDRBBpost:
3915 case AArch64::LDRBpost:
3916 case AArch64::LDRDpost:
3917 case AArch64::LDRHHpost:
3918 case AArch64::LDRHpost:
3919 case AArch64::LDRQpost:
3920 case AArch64::LDRSBWpost:
3921 case AArch64::LDRSBXpost:
3922 case AArch64::LDRSHWpost:
3923 case AArch64::LDRSHXpost:
3924 case AArch64::LDRSWpost:
3925 case AArch64::LDRSpost:
3926 case AArch64::LDRWpost:
3927 case AArch64::LDRXpost:
3928 case AArch64::ST1Fourv16b_POST:
3929 case AArch64::ST1Fourv1d_POST:
3930 case AArch64::ST1Fourv2d_POST:
3931 case AArch64::ST1Fourv2s_POST:
3932 case AArch64::ST1Fourv4h_POST:
3933 case AArch64::ST1Fourv4s_POST:
3934 case AArch64::ST1Fourv8b_POST:
3935 case AArch64::ST1Fourv8h_POST:
3936 case AArch64::ST1Onev16b_POST:
3937 case AArch64::ST1Onev1d_POST:
3938 case AArch64::ST1Onev2d_POST:
3939 case AArch64::ST1Onev2s_POST:
3940 case AArch64::ST1Onev4h_POST:
3941 case AArch64::ST1Onev4s_POST:
3942 case AArch64::ST1Onev8b_POST:
3943 case AArch64::ST1Onev8h_POST:
3944 case AArch64::ST1Threev16b_POST:
3945 case AArch64::ST1Threev1d_POST:
3946 case AArch64::ST1Threev2d_POST:
3947 case AArch64::ST1Threev2s_POST:
3948 case AArch64::ST1Threev4h_POST:
3949 case AArch64::ST1Threev4s_POST:
3950 case AArch64::ST1Threev8b_POST:
3951 case AArch64::ST1Threev8h_POST:
3952 case AArch64::ST1Twov16b_POST:
3953 case AArch64::ST1Twov1d_POST:
3954 case AArch64::ST1Twov2d_POST:
3955 case AArch64::ST1Twov2s_POST:
3956 case AArch64::ST1Twov4h_POST:
3957 case AArch64::ST1Twov4s_POST:
3958 case AArch64::ST1Twov8b_POST:
3959 case AArch64::ST1Twov8h_POST:
3960 case AArch64::ST1i16_POST:
3961 case AArch64::ST1i32_POST:
3962 case AArch64::ST1i64_POST:
3963 case AArch64::ST1i8_POST:
3964 case AArch64::ST2GPostIndex:
3965 case AArch64::ST2Twov16b_POST:
3966 case AArch64::ST2Twov2d_POST:
3967 case AArch64::ST2Twov2s_POST:
3968 case AArch64::ST2Twov4h_POST:
3969 case AArch64::ST2Twov4s_POST:
3970 case AArch64::ST2Twov8b_POST:
3971 case AArch64::ST2Twov8h_POST:
3972 case AArch64::ST2i16_POST:
3973 case AArch64::ST2i32_POST:
3974 case AArch64::ST2i64_POST:
3975 case AArch64::ST2i8_POST:
3976 case AArch64::ST3Threev16b_POST:
3977 case AArch64::ST3Threev2d_POST:
3978 case AArch64::ST3Threev2s_POST:
3979 case AArch64::ST3Threev4h_POST:
3980 case AArch64::ST3Threev4s_POST:
3981 case AArch64::ST3Threev8b_POST:
3982 case AArch64::ST3Threev8h_POST:
3983 case AArch64::ST3i16_POST:
3984 case AArch64::ST3i32_POST:
3985 case AArch64::ST3i64_POST:
3986 case AArch64::ST3i8_POST:
3987 case AArch64::ST4Fourv16b_POST:
3988 case AArch64::ST4Fourv2d_POST:
3989 case AArch64::ST4Fourv2s_POST:
3990 case AArch64::ST4Fourv4h_POST:
3991 case AArch64::ST4Fourv4s_POST:
3992 case AArch64::ST4Fourv8b_POST:
3993 case AArch64::ST4Fourv8h_POST:
3994 case AArch64::ST4i16_POST:
3995 case AArch64::ST4i32_POST:
3996 case AArch64::ST4i64_POST:
3997 case AArch64::ST4i8_POST:
3998 case AArch64::STGPostIndex:
3999 case AArch64::STGPpost:
4000 case AArch64::STPDpost:
4001 case AArch64::STPQpost:
4002 case AArch64::STPSpost:
4003 case AArch64::STPWpost:
4004 case AArch64::STPXpost:
4005 case AArch64::STRBBpost:
4006 case AArch64::STRBpost:
4007 case AArch64::STRDpost:
4008 case AArch64::STRHHpost:
4009 case AArch64::STRHpost:
4010 case AArch64::STRQpost:
4011 case AArch64::STRSpost:
4012 case AArch64::STRWpost:
4013 case AArch64::STRXpost:
4014 case AArch64::STZ2GPostIndex:
4015 case AArch64::STZGPostIndex:
4022 bool &OffsetIsScalable,
TypeSize &Width,
4043 int64_t Dummy1, Dummy2;
4065 return BaseOp->
isReg() || BaseOp->
isFI();
4072 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
4077 TypeSize &Width, int64_t &MinOffset,
4078 int64_t &MaxOffset) {
4084 MinOffset = MaxOffset = 0;
4087 case AArch64::LDRQui:
4088 case AArch64::STRQui:
4094 case AArch64::LDRXui:
4095 case AArch64::LDRDui:
4096 case AArch64::STRXui:
4097 case AArch64::STRDui:
4098 case AArch64::PRFMui:
4104 case AArch64::LDRWui:
4105 case AArch64::LDRSui:
4106 case AArch64::LDRSWui:
4107 case AArch64::STRWui:
4108 case AArch64::STRSui:
4114 case AArch64::LDRHui:
4115 case AArch64::LDRHHui:
4116 case AArch64::LDRSHWui:
4117 case AArch64::LDRSHXui:
4118 case AArch64::STRHui:
4119 case AArch64::STRHHui:
4125 case AArch64::LDRBui:
4126 case AArch64::LDRBBui:
4127 case AArch64::LDRSBWui:
4128 case AArch64::LDRSBXui:
4129 case AArch64::STRBui:
4130 case AArch64::STRBBui:
4137 case AArch64::STRQpre:
4138 case AArch64::LDRQpost:
4144 case AArch64::LDRDpost:
4145 case AArch64::LDRDpre:
4146 case AArch64::LDRXpost:
4147 case AArch64::LDRXpre:
4148 case AArch64::STRDpost:
4149 case AArch64::STRDpre:
4150 case AArch64::STRXpost:
4151 case AArch64::STRXpre:
4157 case AArch64::STRWpost:
4158 case AArch64::STRWpre:
4159 case AArch64::LDRWpost:
4160 case AArch64::LDRWpre:
4161 case AArch64::STRSpost:
4162 case AArch64::STRSpre:
4163 case AArch64::LDRSpost:
4164 case AArch64::LDRSpre:
4170 case AArch64::LDRHpost:
4171 case AArch64::LDRHpre:
4172 case AArch64::STRHpost:
4173 case AArch64::STRHpre:
4174 case AArch64::LDRHHpost:
4175 case AArch64::LDRHHpre:
4176 case AArch64::STRHHpost:
4177 case AArch64::STRHHpre:
4183 case AArch64::LDRBpost:
4184 case AArch64::LDRBpre:
4185 case AArch64::STRBpost:
4186 case AArch64::STRBpre:
4187 case AArch64::LDRBBpost:
4188 case AArch64::LDRBBpre:
4189 case AArch64::STRBBpost:
4190 case AArch64::STRBBpre:
4197 case AArch64::LDURQi:
4198 case AArch64::STURQi:
4204 case AArch64::LDURXi:
4205 case AArch64::LDURDi:
4206 case AArch64::LDAPURXi:
4207 case AArch64::STURXi:
4208 case AArch64::STURDi:
4209 case AArch64::STLURXi:
4210 case AArch64::PRFUMi:
4216 case AArch64::LDURWi:
4217 case AArch64::LDURSi:
4218 case AArch64::LDURSWi:
4219 case AArch64::LDAPURi:
4220 case AArch64::LDAPURSWi:
4221 case AArch64::STURWi:
4222 case AArch64::STURSi:
4223 case AArch64::STLURWi:
4229 case AArch64::LDURHi:
4230 case AArch64::LDURHHi:
4231 case AArch64::LDURSHXi:
4232 case AArch64::LDURSHWi:
4233 case AArch64::LDAPURHi:
4234 case AArch64::LDAPURSHWi:
4235 case AArch64::LDAPURSHXi:
4236 case AArch64::STURHi:
4237 case AArch64::STURHHi:
4238 case AArch64::STLURHi:
4244 case AArch64::LDURBi:
4245 case AArch64::LDURBBi:
4246 case AArch64::LDURSBXi:
4247 case AArch64::LDURSBWi:
4248 case AArch64::LDAPURBi:
4249 case AArch64::LDAPURSBWi:
4250 case AArch64::LDAPURSBXi:
4251 case AArch64::STURBi:
4252 case AArch64::STURBBi:
4253 case AArch64::STLURBi:
4260 case AArch64::LDPQi:
4261 case AArch64::LDNPQi:
4262 case AArch64::STPQi:
4263 case AArch64::STNPQi:
4264 case AArch64::LDPQpost:
4265 case AArch64::LDPQpre:
4266 case AArch64::STPQpost:
4267 case AArch64::STPQpre:
4273 case AArch64::LDPXi:
4274 case AArch64::LDPDi:
4275 case AArch64::LDNPXi:
4276 case AArch64::LDNPDi:
4277 case AArch64::STPXi:
4278 case AArch64::STPDi:
4279 case AArch64::STNPXi:
4280 case AArch64::STNPDi:
4281 case AArch64::LDPDpost:
4282 case AArch64::LDPDpre:
4283 case AArch64::LDPXpost:
4284 case AArch64::LDPXpre:
4285 case AArch64::STPDpost:
4286 case AArch64::STPDpre:
4287 case AArch64::STPXpost:
4288 case AArch64::STPXpre:
4294 case AArch64::LDPWi:
4295 case AArch64::LDPSi:
4296 case AArch64::LDNPWi:
4297 case AArch64::LDNPSi:
4298 case AArch64::STPWi:
4299 case AArch64::STPSi:
4300 case AArch64::STNPWi:
4301 case AArch64::STNPSi:
4302 case AArch64::LDPSpost:
4303 case AArch64::LDPSpre:
4304 case AArch64::LDPWpost:
4305 case AArch64::LDPWpre:
4306 case AArch64::STPSpost:
4307 case AArch64::STPSpre:
4308 case AArch64::STPWpost:
4309 case AArch64::STPWpre:
4315 case AArch64::StoreSwiftAsyncContext:
4328 case AArch64::TAGPstack:
4338 case AArch64::STGPreIndex:
4339 case AArch64::STGPostIndex:
4340 case AArch64::STZGi:
4341 case AArch64::STZGPreIndex:
4342 case AArch64::STZGPostIndex:
4349 case AArch64::STR_ZZZZXI:
4350 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
4351 case AArch64::LDR_ZZZZXI:
4352 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
4358 case AArch64::STR_ZZZXI:
4359 case AArch64::LDR_ZZZXI:
4365 case AArch64::STR_ZZXI:
4366 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
4367 case AArch64::LDR_ZZXI:
4368 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
4374 case AArch64::LDR_PXI:
4375 case AArch64::STR_PXI:
4381 case AArch64::LDR_PPXI:
4382 case AArch64::STR_PPXI:
4388 case AArch64::LDR_ZXI:
4389 case AArch64::STR_ZXI:
4395 case AArch64::LD1B_IMM:
4396 case AArch64::LD1H_IMM:
4397 case AArch64::LD1W_IMM:
4398 case AArch64::LD1D_IMM:
4399 case AArch64::LDNT1B_ZRI:
4400 case AArch64::LDNT1H_ZRI:
4401 case AArch64::LDNT1W_ZRI:
4402 case AArch64::LDNT1D_ZRI:
4403 case AArch64::ST1B_IMM:
4404 case AArch64::ST1H_IMM:
4405 case AArch64::ST1W_IMM:
4406 case AArch64::ST1D_IMM:
4407 case AArch64::STNT1B_ZRI:
4408 case AArch64::STNT1H_ZRI:
4409 case AArch64::STNT1W_ZRI:
4410 case AArch64::STNT1D_ZRI:
4411 case AArch64::LDNF1B_IMM:
4412 case AArch64::LDNF1H_IMM:
4413 case AArch64::LDNF1W_IMM:
4414 case AArch64::LDNF1D_IMM:
4422 case AArch64::LD2B_IMM:
4423 case AArch64::LD2H_IMM:
4424 case AArch64::LD2W_IMM:
4425 case AArch64::LD2D_IMM:
4426 case AArch64::ST2B_IMM:
4427 case AArch64::ST2H_IMM:
4428 case AArch64::ST2W_IMM:
4429 case AArch64::ST2D_IMM:
4435 case AArch64::LD3B_IMM:
4436 case AArch64::LD3H_IMM:
4437 case AArch64::LD3W_IMM:
4438 case AArch64::LD3D_IMM:
4439 case AArch64::ST3B_IMM:
4440 case AArch64::ST3H_IMM:
4441 case AArch64::ST3W_IMM:
4442 case AArch64::ST3D_IMM:
4448 case AArch64::LD4B_IMM:
4449 case AArch64::LD4H_IMM:
4450 case AArch64::LD4W_IMM:
4451 case AArch64::LD4D_IMM:
4452 case AArch64::ST4B_IMM:
4453 case AArch64::ST4H_IMM:
4454 case AArch64::ST4W_IMM:
4455 case AArch64::ST4D_IMM:
4461 case AArch64::LD1B_H_IMM:
4462 case AArch64::LD1SB_H_IMM:
4463 case AArch64::LD1H_S_IMM:
4464 case AArch64::LD1SH_S_IMM:
4465 case AArch64::LD1W_D_IMM:
4466 case AArch64::LD1SW_D_IMM:
4467 case AArch64::ST1B_H_IMM:
4468 case AArch64::ST1H_S_IMM:
4469 case AArch64::ST1W_D_IMM:
4470 case AArch64::LDNF1B_H_IMM:
4471 case AArch64::LDNF1SB_H_IMM:
4472 case AArch64::LDNF1H_S_IMM:
4473 case AArch64::LDNF1SH_S_IMM:
4474 case AArch64::LDNF1W_D_IMM:
4475 case AArch64::LDNF1SW_D_IMM:
4483 case AArch64::LD1B_S_IMM:
4484 case AArch64::LD1SB_S_IMM:
4485 case AArch64::LD1H_D_IMM:
4486 case AArch64::LD1SH_D_IMM:
4487 case AArch64::ST1B_S_IMM:
4488 case AArch64::ST1H_D_IMM:
4489 case AArch64::LDNF1B_S_IMM:
4490 case AArch64::LDNF1SB_S_IMM:
4491 case AArch64::LDNF1H_D_IMM:
4492 case AArch64::LDNF1SH_D_IMM:
4500 case AArch64::LD1B_D_IMM:
4501 case AArch64::LD1SB_D_IMM:
4502 case AArch64::ST1B_D_IMM:
4503 case AArch64::LDNF1B_D_IMM:
4504 case AArch64::LDNF1SB_D_IMM:
4512 case AArch64::ST2Gi:
4513 case AArch64::ST2GPreIndex:
4514 case AArch64::ST2GPostIndex:
4515 case AArch64::STZ2Gi:
4516 case AArch64::STZ2GPreIndex:
4517 case AArch64::STZ2GPostIndex:
4523 case AArch64::STGPi:
4524 case AArch64::STGPpost:
4525 case AArch64::STGPpre:
4531 case AArch64::LD1RB_IMM:
4532 case AArch64::LD1RB_H_IMM:
4533 case AArch64::LD1RB_S_IMM:
4534 case AArch64::LD1RB_D_IMM:
4535 case AArch64::LD1RSB_H_IMM:
4536 case AArch64::LD1RSB_S_IMM:
4537 case AArch64::LD1RSB_D_IMM:
4543 case AArch64::LD1RH_IMM:
4544 case AArch64::LD1RH_S_IMM:
4545 case AArch64::LD1RH_D_IMM:
4546 case AArch64::LD1RSH_S_IMM:
4547 case AArch64::LD1RSH_D_IMM:
4553 case AArch64::LD1RW_IMM:
4554 case AArch64::LD1RW_D_IMM:
4555 case AArch64::LD1RSW_IMM:
4561 case AArch64::LD1RD_IMM:
4577 case AArch64::LDRBBui:
4578 case AArch64::LDURBBi:
4579 case AArch64::LDRSBWui:
4580 case AArch64::LDURSBWi:
4581 case AArch64::STRBBui:
4582 case AArch64::STURBBi:
4584 case AArch64::LDRHHui:
4585 case AArch64::LDURHHi:
4586 case AArch64::LDRSHWui:
4587 case AArch64::LDURSHWi:
4588 case AArch64::STRHHui:
4589 case AArch64::STURHHi:
4591 case AArch64::LDRSui:
4592 case AArch64::LDURSi:
4593 case AArch64::LDRSpre:
4594 case AArch64::LDRSWui:
4595 case AArch64::LDURSWi:
4596 case AArch64::LDRSWpre:
4597 case AArch64::LDRWpre:
4598 case AArch64::LDRWui:
4599 case AArch64::LDURWi:
4600 case AArch64::STRSui:
4601 case AArch64::STURSi:
4602 case AArch64::STRSpre:
4603 case AArch64::STRWui:
4604 case AArch64::STURWi:
4605 case AArch64::STRWpre:
4606 case AArch64::LDPSi:
4607 case AArch64::LDPSWi:
4608 case AArch64::LDPWi:
4609 case AArch64::STPSi:
4610 case AArch64::STPWi:
4612 case AArch64::LDRDui:
4613 case AArch64::LDURDi:
4614 case AArch64::LDRDpre:
4615 case AArch64::LDRXui:
4616 case AArch64::LDURXi:
4617 case AArch64::LDRXpre:
4618 case AArch64::STRDui:
4619 case AArch64::STURDi:
4620 case AArch64::STRDpre:
4621 case AArch64::STRXui:
4622 case AArch64::STURXi:
4623 case AArch64::STRXpre:
4624 case AArch64::LDPDi:
4625 case AArch64::LDPXi:
4626 case AArch64::STPDi:
4627 case AArch64::STPXi:
4629 case AArch64::LDRQui:
4630 case AArch64::LDURQi:
4631 case AArch64::STRQui:
4632 case AArch64::STURQi:
4633 case AArch64::STRQpre:
4634 case AArch64::LDPQi:
4635 case AArch64::LDRQpre:
4636 case AArch64::STPQi:
4638 case AArch64::STZGi:
4639 case AArch64::ST2Gi:
4640 case AArch64::STZ2Gi:
4641 case AArch64::STGPi:
4647 switch (
MI.getOpcode()) {
4650 case AArch64::LDRWpre:
4651 case AArch64::LDRXpre:
4652 case AArch64::LDRSWpre:
4653 case AArch64::LDRSpre:
4654 case AArch64::LDRDpre:
4655 case AArch64::LDRQpre:
4661 switch (
MI.getOpcode()) {
4664 case AArch64::STRWpre:
4665 case AArch64::STRXpre:
4666 case AArch64::STRSpre:
4667 case AArch64::STRDpre:
4668 case AArch64::STRQpre:
4678 switch (
MI.getOpcode()) {
4681 case AArch64::LDPSi:
4682 case AArch64::LDPSWi:
4683 case AArch64::LDPDi:
4684 case AArch64::LDPQi:
4685 case AArch64::LDPWi:
4686 case AArch64::LDPXi:
4687 case AArch64::STPSi:
4688 case AArch64::STPDi:
4689 case AArch64::STPQi:
4690 case AArch64::STPWi:
4691 case AArch64::STPXi:
4692 case AArch64::STGPi:
4698 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4702 return MI.getOperand(Idx);
4707 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4711 return MI.getOperand(Idx);
4716 switch (
MI.getOpcode()) {
4719 case AArch64::LDRBroX:
4720 case AArch64::LDRBBroX:
4721 case AArch64::LDRSBXroX:
4722 case AArch64::LDRSBWroX:
4723 case AArch64::LDRHroX:
4724 case AArch64::LDRHHroX:
4725 case AArch64::LDRSHXroX:
4726 case AArch64::LDRSHWroX:
4727 case AArch64::LDRWroX:
4728 case AArch64::LDRSroX:
4729 case AArch64::LDRSWroX:
4730 case AArch64::LDRDroX:
4731 case AArch64::LDRXroX:
4732 case AArch64::LDRQroX:
4733 return MI.getOperand(4);
4739 if (
MI.getParent() ==
nullptr)
4749 auto Reg =
Op.getReg();
4750 if (Reg.isPhysical())
4751 return AArch64::FPR16RegClass.contains(Reg);
4753 return TRC == &AArch64::FPR16RegClass ||
4754 TRC == &AArch64::FPR16_loRegClass;
4763 auto Reg =
Op.getReg();
4764 if (Reg.isPhysical())
4765 return AArch64::FPR128RegClass.contains(Reg);
4767 return TRC == &AArch64::FPR128RegClass ||
4768 TRC == &AArch64::FPR128_loRegClass;
4774 switch (
MI.getOpcode()) {
4777 case AArch64::PACIASP:
4778 case AArch64::PACIBSP:
4781 case AArch64::PAUTH_PROLOGUE:
4784 case AArch64::HINT: {
4785 unsigned Imm =
MI.getOperand(0).getImm();
4787 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4790 if (Imm == 25 || Imm == 27)
4802 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4803 return AArch64::FPR128RegClass.contains(Reg) ||
4804 AArch64::FPR64RegClass.contains(Reg) ||
4805 AArch64::FPR32RegClass.contains(Reg) ||
4806 AArch64::FPR16RegClass.contains(Reg) ||
4807 AArch64::FPR8RegClass.contains(Reg);
4814 auto Reg =
Op.getReg();
4815 if (Reg.isPhysical())
4819 return TRC == &AArch64::FPR128RegClass ||
4820 TRC == &AArch64::FPR128_loRegClass ||
4821 TRC == &AArch64::FPR64RegClass ||
4822 TRC == &AArch64::FPR64_loRegClass ||
4823 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4824 TRC == &AArch64::FPR8RegClass;
4846 if (FirstOpc == SecondOpc)
4852 case AArch64::STRSui:
4853 case AArch64::STURSi:
4854 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4855 case AArch64::STRDui:
4856 case AArch64::STURDi:
4857 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4858 case AArch64::STRQui:
4859 case AArch64::STURQi:
4860 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4861 case AArch64::STRWui:
4862 case AArch64::STURWi:
4863 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4864 case AArch64::STRXui:
4865 case AArch64::STURXi:
4866 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4867 case AArch64::LDRSui:
4868 case AArch64::LDURSi:
4869 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4870 case AArch64::LDRDui:
4871 case AArch64::LDURDi:
4872 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4873 case AArch64::LDRQui:
4874 case AArch64::LDURQi:
4875 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4876 case AArch64::LDRWui:
4877 case AArch64::LDURWi:
4878 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4879 case AArch64::LDRSWui:
4880 case AArch64::LDURSWi:
4881 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4882 case AArch64::LDRXui:
4883 case AArch64::LDURXi:
4884 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4891 int64_t Offset1,
unsigned Opcode1,
int FI2,
4892 int64_t Offset2,
unsigned Opcode2) {
4898 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4901 if (ObjectOffset1 % Scale1 != 0)
4903 ObjectOffset1 /= Scale1;
4905 if (ObjectOffset2 % Scale2 != 0)
4907 ObjectOffset2 /= Scale2;
4908 ObjectOffset1 += Offset1;
4909 ObjectOffset2 += Offset2;
4910 return ObjectOffset1 + 1 == ObjectOffset2;
4922 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4923 unsigned NumBytes)
const {
4933 "Only base registers and frame indices are supported.");
4940 if (ClusterSize > 2)
4947 unsigned FirstOpc = FirstLdSt.
getOpcode();
4948 unsigned SecondOpc = SecondLdSt.
getOpcode();
4968 if (Offset1 > 63 || Offset1 < -64)
4973 if (BaseOp1.
isFI()) {
4975 "Caller should have ordered offsets.");
4980 BaseOp2.
getIndex(), Offset2, SecondOpc);
4983 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4985 return Offset1 + 1 == Offset2;
4995 if (
Reg.isPhysical())
5004 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
5013 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
5015 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
5016 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
5017 unsigned NumRegs = Indices.
size();
5019 int SubReg = 0, End = NumRegs, Incr = 1;
5038 unsigned Opcode,
unsigned ZeroReg,
5041 unsigned NumRegs = Indices.
size();
5044 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
5045 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
5046 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5047 "GPR reg sequences should not be able to overlap");
5064 bool RenamableSrc)
const {
5065 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
5066 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
5069 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5071 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5072 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5075 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
5077 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
5093 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGPR32()) {
5097 }
else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5098 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5100 MCRegister DestRegX =
TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
5101 &AArch64::GPR64spRegClass);
5102 assert(DestRegX.
isValid() &&
"Destination super-reg not valid");
5104 SrcReg == AArch64::WZR
5106 :
TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
5107 &AArch64::GPR64spRegClass);
5127 if (AArch64::PPRRegClass.
contains(DestReg) &&
5128 AArch64::PPRRegClass.
contains(SrcReg)) {
5129 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5130 "Unexpected SVE register.");
5140 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
5141 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
5142 if (DestIsPNR || SrcIsPNR) {
5144 return (R - AArch64::PN0) + AArch64::P0;
5149 if (PPRSrcReg != PPRDestReg) {
5161 if (AArch64::ZPRRegClass.
contains(DestReg) &&
5162 AArch64::ZPRRegClass.
contains(SrcReg)) {
5163 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5164 "Unexpected SVE register.");
5172 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
5173 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
5174 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
5175 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
5176 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5177 "Unexpected SVE register.");
5178 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5185 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
5186 AArch64::ZPR3RegClass.
contains(SrcReg)) {
5187 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5188 "Unexpected SVE register.");
5189 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5197 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
5198 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
5199 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
5200 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5201 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5202 "Unexpected SVE register.");
5203 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5204 AArch64::zsub2, AArch64::zsub3};
5210 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5211 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
5212 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5218 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGPR64()) {
5232 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5233 AArch64::DDDDRegClass.
contains(SrcReg)) {
5234 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5235 AArch64::dsub2, AArch64::dsub3};
5242 if (AArch64::DDDRegClass.
contains(DestReg) &&
5243 AArch64::DDDRegClass.
contains(SrcReg)) {
5244 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5252 if (AArch64::DDRegClass.
contains(DestReg) &&
5253 AArch64::DDRegClass.
contains(SrcReg)) {
5254 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5261 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5262 AArch64::QQQQRegClass.
contains(SrcReg)) {
5263 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5264 AArch64::qsub2, AArch64::qsub3};
5271 if (AArch64::QQQRegClass.
contains(DestReg) &&
5272 AArch64::QQQRegClass.
contains(SrcReg)) {
5273 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5281 if (AArch64::QQRegClass.
contains(DestReg) &&
5282 AArch64::QQRegClass.
contains(SrcReg)) {
5283 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5289 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5290 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5291 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5293 AArch64::XZR, Indices);
5297 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5298 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5299 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5301 AArch64::WZR, Indices);
5305 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5306 AArch64::FPR128RegClass.
contains(SrcReg)) {
5307 if (Subtarget.isSVEorStreamingSVEAvailable() &&
5308 !Subtarget.isNeonAvailable())
5311 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5312 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5313 else if (Subtarget.isNeonAvailable())
5332 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5333 AArch64::FPR64RegClass.
contains(SrcReg)) {
5334 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5335 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5336 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5338 MCRegister DestRegQ =
TRI->getMatchingSuperReg(DestReg, AArch64::dsub,
5339 &AArch64::FPR128RegClass);
5340 MCRegister SrcRegQ =
TRI->getMatchingSuperReg(SrcReg, AArch64::dsub,
5341 &AArch64::FPR128RegClass);
5357 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5358 AArch64::FPR32RegClass.
contains(SrcReg)) {
5359 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5360 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5361 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5363 MCRegister DestRegQ =
TRI->getMatchingSuperReg(DestReg, AArch64::ssub,
5364 &AArch64::FPR128RegClass);
5365 MCRegister SrcRegQ =
TRI->getMatchingSuperReg(SrcReg, AArch64::ssub,
5366 &AArch64::FPR128RegClass);
5375 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5376 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5378 MCRegister DestRegD =
TRI->getMatchingSuperReg(DestReg, AArch64::ssub,
5379 &AArch64::FPR64RegClass);
5380 MCRegister SrcRegD =
TRI->getMatchingSuperReg(SrcReg, AArch64::ssub,
5381 &AArch64::FPR64RegClass);
5396 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5397 AArch64::FPR16RegClass.
contains(SrcReg)) {
5398 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5399 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5400 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5402 MCRegister DestRegQ =
TRI->getMatchingSuperReg(DestReg, AArch64::hsub,
5403 &AArch64::FPR128RegClass);
5404 MCRegister SrcRegQ =
TRI->getMatchingSuperReg(SrcReg, AArch64::hsub,
5405 &AArch64::FPR128RegClass);
5414 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5415 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5417 MCRegister DestRegD =
TRI->getMatchingSuperReg(DestReg, AArch64::hsub,
5418 &AArch64::FPR64RegClass);
5419 MCRegister SrcRegD =
TRI->getMatchingSuperReg(SrcReg, AArch64::hsub,
5420 &AArch64::FPR64RegClass);
5429 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5430 &AArch64::FPR32RegClass);
5431 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5432 &AArch64::FPR32RegClass);
5439 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5440 AArch64::FPR8RegClass.
contains(SrcReg)) {
5441 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5442 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5443 !Subtarget.hasZeroCycleRegMoveFPR64() && Subtarget.isNeonAvailable()) {
5445 MCRegister DestRegQ =
TRI->getMatchingSuperReg(DestReg, AArch64::bsub,
5446 &AArch64::FPR128RegClass);
5447 MCRegister SrcRegQ =
TRI->getMatchingSuperReg(SrcReg, AArch64::bsub,
5448 &AArch64::FPR128RegClass);
5457 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5458 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5460 MCRegister DestRegD =
TRI->getMatchingSuperReg(DestReg, AArch64::bsub,
5461 &AArch64::FPR64RegClass);
5462 MCRegister SrcRegD =
TRI->getMatchingSuperReg(SrcReg, AArch64::bsub,
5463 &AArch64::FPR64RegClass);
5472 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5473 &AArch64::FPR32RegClass);
5474 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5475 &AArch64::FPR32RegClass);
5483 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5484 AArch64::GPR64RegClass.
contains(SrcReg)) {
5485 if (AArch64::XZR == SrcReg) {
5493 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5494 AArch64::FPR64RegClass.
contains(SrcReg)) {
5500 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5501 AArch64::GPR32RegClass.
contains(SrcReg)) {
5502 if (AArch64::WZR == SrcReg) {
5510 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5511 AArch64::FPR32RegClass.
contains(SrcReg)) {
5517 if (DestReg == AArch64::NZCV) {
5518 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5520 .
addImm(AArch64SysReg::NZCV)
5526 if (SrcReg == AArch64::NZCV) {
5527 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
5529 .
addImm(AArch64SysReg::NZCV)
5536 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
5537 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
5547 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
5552 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
5554 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
5567 Register SrcReg,
bool isKill,
int FI,
5583 switch (
TRI->getSpillSize(*RC)) {
5585 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5586 Opc = AArch64::STRBui;
5589 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5590 Opc = AArch64::STRHui;
5591 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
5592 AArch64::PPRRegClass.hasSubClassEq(RC)) {
5593 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5594 "Unexpected register store without SVE store instructions");
5595 Opc = AArch64::STR_PXI;
5601 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5602 Opc = AArch64::STRWui;
5606 assert(SrcReg != AArch64::WSP);
5607 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5608 Opc = AArch64::STRSui;
5609 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5610 Opc = AArch64::STR_PPXI;
5615 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5616 Opc = AArch64::STRXui;
5620 assert(SrcReg != AArch64::SP);
5621 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5622 Opc = AArch64::STRDui;
5623 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5625 get(AArch64::STPWi), SrcReg, isKill,
5626 AArch64::sube32, AArch64::subo32, FI, MMO);
5631 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5632 Opc = AArch64::STRQui;
5633 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5634 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5635 Opc = AArch64::ST1Twov1d;
5637 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5639 get(AArch64::STPXi), SrcReg, isKill,
5640 AArch64::sube64, AArch64::subo64, FI, MMO);
5642 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5643 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5644 "Unexpected register store without SVE store instructions");
5645 Opc = AArch64::STR_ZXI;
5650 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5651 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5652 Opc = AArch64::ST1Threev1d;
5657 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5658 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5659 Opc = AArch64::ST1Fourv1d;
5661 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5662 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5663 Opc = AArch64::ST1Twov2d;
5665 }
else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5666 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5667 "Unexpected register store without SVE store instructions");
5668 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
5670 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
5671 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5672 "Unexpected register store without SVE store instructions");
5673 Opc = AArch64::STR_ZZXI;
5678 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5679 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5680 Opc = AArch64::ST1Threev2d;
5682 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5683 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5684 "Unexpected register store without SVE store instructions");
5685 Opc = AArch64::STR_ZZZXI;
5690 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5691 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5692 Opc = AArch64::ST1Fourv2d;
5694 }
else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5695 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5696 "Unexpected register store without SVE store instructions");
5697 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
5699 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
5700 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5701 "Unexpected register store without SVE store instructions");
5702 Opc = AArch64::STR_ZZZZXI;
5707 assert(
Opc &&
"Unknown register class");
5718 MI.addMemOperand(MMO);
5725 Register DestReg,
unsigned SubIdx0,
5726 unsigned SubIdx1,
int FI,
5730 bool IsUndef =
true;
5732 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
5734 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5761 switch (
TRI->getSpillSize(*RC)) {
5763 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5764 Opc = AArch64::LDRBui;
5767 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5768 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5769 Opc = AArch64::LDRHui;
5770 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5771 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5772 "Unexpected register load without SVE load instructions");
5775 Opc = AArch64::LDR_PXI;
5781 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5782 Opc = AArch64::LDRWui;
5786 assert(DestReg != AArch64::WSP);
5787 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5788 Opc = AArch64::LDRSui;
5789 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5790 Opc = AArch64::LDR_PPXI;
5795 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5796 Opc = AArch64::LDRXui;
5800 assert(DestReg != AArch64::SP);
5801 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5802 Opc = AArch64::LDRDui;
5803 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5805 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5806 AArch64::subo32, FI, MMO);
5811 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5812 Opc = AArch64::LDRQui;
5813 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5814 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5815 Opc = AArch64::LD1Twov1d;
5817 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5819 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5820 AArch64::subo64, FI, MMO);
5822 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5823 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5824 "Unexpected register load without SVE load instructions");
5825 Opc = AArch64::LDR_ZXI;
5830 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5831 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5832 Opc = AArch64::LD1Threev1d;
5837 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5838 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5839 Opc = AArch64::LD1Fourv1d;
5841 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5842 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5843 Opc = AArch64::LD1Twov2d;
5845 }
else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5846 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5847 "Unexpected register load without SVE load instructions");
5848 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
5850 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
5851 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5852 "Unexpected register load without SVE load instructions");
5853 Opc = AArch64::LDR_ZZXI;
5858 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5859 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5860 Opc = AArch64::LD1Threev2d;
5862 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5863 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5864 "Unexpected register load without SVE load instructions");
5865 Opc = AArch64::LDR_ZZZXI;
5870 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5871 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5872 Opc = AArch64::LD1Fourv2d;
5874 }
else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5875 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5876 "Unexpected register load without SVE load instructions");
5877 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
5879 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
5880 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5881 "Unexpected register load without SVE load instructions");
5882 Opc = AArch64::LDR_ZZZZXI;
5888 assert(
Opc &&
"Unknown register class");
5898 MI.addMemOperand(MMO);
5905 UseMI.getIterator()),
5907 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5908 I.readsRegister(AArch64::NZCV, TRI);
5912void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
5917 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5924 ByteSized =
Offset.getFixed();
5925 VGSized =
Offset.getScalable() / 2;
5931void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
5933 int64_t &NumDataVectors) {
5937 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5939 NumBytes =
Offset.getFixed();
5941 NumPredicateVectors =
Offset.getScalable() / 2;
5946 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5947 NumPredicateVectors > 62) {
5948 NumDataVectors = NumPredicateVectors / 8;
5949 NumPredicateVectors -= NumDataVectors * 8;
5975 Expr.
push_back((
char)dwarf::DW_OP_bregx);
5983 int64_t OffsetFromDefCFA) {
5997 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5998 if (!RegScale.empty())
6008 int64_t NumBytes, NumVGScaledBytes;
6009 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
Offset, NumBytes,
6011 std::string CommentBuffer;
6014 if (
Reg == AArch64::SP)
6016 else if (
Reg == AArch64::FP)
6023 unsigned DwarfReg =
TRI.getDwarfRegNum(
Reg,
true);
6024 assert(DwarfReg <= 31 &&
"DwarfReg out of bounds (0..31)");
6026 Expr.
push_back(dwarf::DW_OP_breg0 + DwarfReg);
6029 if (NumVGScaledBytes) {
6039 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
6047 unsigned FrameReg,
unsigned Reg,
6049 bool LastAdjustmentWasScalable) {
6050 if (
Offset.getScalable())
6053 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6056 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
6063 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6064 int64_t NumBytes, NumVGScaledBytes;
6065 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6066 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
6068 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
6071 if (!NumVGScaledBytes)
6074 std::string CommentBuffer;
6079 assert(NumVGScaledBytes &&
"Expected scalable offset");
6083 if (IncomingVGOffsetFromDefCFA) {
6085 VGRegScale =
"* IncomingVG";
6088 VGRegScale =
"* VG";
6092 OffsetExpr.
push_back(dwarf::DW_OP_plus);
6101 CfaExpr.
push_back(dwarf::DW_CFA_expression);
6116 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
6119 bool *HasWinCFI,
bool EmitCFAOffset,
6122 unsigned MaxEncoding, ShiftSize;
6124 case AArch64::ADDXri:
6125 case AArch64::ADDSXri:
6126 case AArch64::SUBXri:
6127 case AArch64::SUBSXri:
6128 MaxEncoding = 0xfff;
6131 case AArch64::ADDVL_XXI:
6132 case AArch64::ADDPL_XXI:
6133 case AArch64::ADDSVL_XXI:
6134 case AArch64::ADDSPL_XXI:
6149 if (
Opc == AArch64::ADDVL_XXI ||
Opc == AArch64::ADDSVL_XXI)
6151 else if (
Opc == AArch64::ADDPL_XXI ||
Opc == AArch64::ADDSPL_XXI)
6165 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
6167 if (TmpReg == AArch64::XZR)
6168 TmpReg =
MBB.getParent()->getRegInfo().createVirtualRegister(
6169 &AArch64::GPR64RegClass);
6171 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
6172 unsigned LocalShiftSize = 0;
6173 if (ThisVal > MaxEncoding) {
6174 ThisVal = ThisVal >> ShiftSize;
6175 LocalShiftSize = ShiftSize;
6177 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
6178 "Encoding cannot handle value that big");
6180 Offset -= ThisVal << LocalShiftSize;
6185 .
addImm(Sign * (
int)ThisVal);
6195 if (Sign == -1 ||
Opc == AArch64::SUBXri ||
Opc == AArch64::SUBSXri)
6196 CFAOffset += Change;
6198 CFAOffset -= Change;
6199 if (EmitCFAOffset && DestReg == TmpReg) {
6212 int Imm = (int)(ThisVal << LocalShiftSize);
6213 if (VScale != 1 && DestReg == AArch64::SP) {
6219 }
else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
6220 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
6221 assert(VScale == 1 &&
"Expected non-scalable operation");
6230 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
6231 "emit a single SEH directive");
6232 }
else if (DestReg == AArch64::SP) {
6233 assert(VScale == 1 &&
"Expected non-scalable operation");
6236 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
6249 unsigned DestReg,
unsigned SrcReg,
6252 bool NeedsWinCFI,
bool *HasWinCFI,
6254 unsigned FrameReg) {
6261 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
6263 int64_t Bytes, NumPredicateVectors, NumDataVectors;
6264 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6265 Offset, Bytes, NumPredicateVectors, NumDataVectors);
6268 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
6269 if (NeedsFinalDefNZCV)
6273 if (Bytes || (!
Offset && SrcReg != DestReg)) {
6274 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
6275 "SP increment/decrement not 8-byte aligned");
6276 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
6279 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
6282 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6284 CFAOffset += (
Opc == AArch64::ADDXri ||
Opc == AArch64::ADDSXri)
6291 assert(!(NeedsWinCFI && NumPredicateVectors) &&
6292 "WinCFI can't allocate fractions of an SVE data vector");
6294 if (NumDataVectors) {
6296 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
TII,
6297 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6303 if (NumPredicateVectors) {
6304 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
6306 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
TII,
6307 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6311 if (NeedsFinalDefNZCV)
6332 if (
MI.isFullCopy()) {
6335 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
6339 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
6344 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
6372 if (
MI.isCopy() &&
Ops.size() == 1 &&
6374 (
Ops[0] == 0 ||
Ops[0] == 1)) {
6375 bool IsSpill =
Ops[0] == 0;
6376 bool IsFill = !IsSpill;
6388 :
TRI.getMinimalPhysRegClass(Reg);
6394 "Mismatched register size in non subreg COPY");
6401 return &*--InsertPt;
6413 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
6416 "Unexpected subreg on physical register");
6418 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
6420 return &*--InsertPt;
6437 case AArch64::sub_32:
6438 if (AArch64::GPR64RegClass.hasSubClassEq(
getRegClass(DstReg)))
6439 FillRC = &AArch64::GPR32RegClass;
6442 FillRC = &AArch64::FPR32RegClass;
6445 FillRC = &AArch64::FPR64RegClass;
6451 TRI.getRegSizeInBits(*FillRC) &&
6452 "Mismatched regclass size on folded subreg COPY");
6471 bool *OutUseUnscaledOp,
6472 unsigned *OutUnscaledOp,
6473 int64_t *EmittableOffset) {
6475 if (EmittableOffset)
6476 *EmittableOffset = 0;
6477 if (OutUseUnscaledOp)
6478 *OutUseUnscaledOp =
false;
6484 switch (
MI.getOpcode()) {
6487 case AArch64::LD1Rv1d:
6488 case AArch64::LD1Rv2s:
6489 case AArch64::LD1Rv2d:
6490 case AArch64::LD1Rv4h:
6491 case AArch64::LD1Rv4s:
6492 case AArch64::LD1Rv8b:
6493 case AArch64::LD1Rv8h:
6494 case AArch64::LD1Rv16b:
6495 case AArch64::LD1Twov2d:
6496 case AArch64::LD1Threev2d:
6497 case AArch64::LD1Fourv2d:
6498 case AArch64::LD1Twov1d:
6499 case AArch64::LD1Threev1d:
6500 case AArch64::LD1Fourv1d:
6501 case AArch64::ST1Twov2d:
6502 case AArch64::ST1Threev2d:
6503 case AArch64::ST1Fourv2d:
6504 case AArch64::ST1Twov1d:
6505 case AArch64::ST1Threev1d:
6506 case AArch64::ST1Fourv1d:
6507 case AArch64::ST1i8:
6508 case AArch64::ST1i16:
6509 case AArch64::ST1i32:
6510 case AArch64::ST1i64:
6512 case AArch64::IRGstack:
6513 case AArch64::STGloop:
6514 case AArch64::STZGloop:
6519 TypeSize ScaleValue(0U,
false), Width(0U,
false);
6520 int64_t MinOff, MaxOff;
6526 bool IsMulVL = ScaleValue.isScalable();
6527 unsigned Scale = ScaleValue.getKnownMinValue();
6537 std::optional<unsigned> UnscaledOp =
6539 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
6540 if (useUnscaledOp &&
6545 Scale = ScaleValue.getKnownMinValue();
6546 assert(IsMulVL == ScaleValue.isScalable() &&
6547 "Unscaled opcode has different value for scalable");
6549 int64_t Remainder =
Offset % Scale;
6550 assert(!(Remainder && useUnscaledOp) &&
6551 "Cannot have remainder when using unscaled op");
6553 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
6554 int64_t NewOffset =
Offset / Scale;
6555 if (MinOff <= NewOffset && NewOffset <= MaxOff)
6558 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
6562 if (EmittableOffset)
6563 *EmittableOffset = NewOffset;
6564 if (OutUseUnscaledOp)
6565 *OutUseUnscaledOp = useUnscaledOp;
6566 if (OutUnscaledOp && UnscaledOp)
6567 *OutUnscaledOp = *UnscaledOp;
6580 unsigned Opcode =
MI.getOpcode();
6581 unsigned ImmIdx = FrameRegIdx + 1;
6583 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
6588 MI.eraseFromParent();
6594 unsigned UnscaledOp;
6597 &UnscaledOp, &NewOffset);
6601 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
6603 MI.setDesc(
TII->get(UnscaledOp));
6605 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
6623bool AArch64InstrInfo::useMachineCombiner()
const {
return true; }
6628 case AArch64::ADDSWrr:
6629 case AArch64::ADDSWri:
6630 case AArch64::ADDSXrr:
6631 case AArch64::ADDSXri:
6632 case AArch64::SUBSWrr:
6633 case AArch64::SUBSXrr:
6635 case AArch64::SUBSWri:
6636 case AArch64::SUBSXri:
6647 case AArch64::ADDWrr:
6648 case AArch64::ADDWri:
6649 case AArch64::SUBWrr:
6650 case AArch64::ADDSWrr:
6651 case AArch64::ADDSWri:
6652 case AArch64::SUBSWrr:
6654 case AArch64::SUBWri:
6655 case AArch64::SUBSWri:
6666 case AArch64::ADDXrr:
6667 case AArch64::ADDXri:
6668 case AArch64::SUBXrr:
6669 case AArch64::ADDSXrr:
6670 case AArch64::ADDSXri:
6671 case AArch64::SUBSXrr:
6673 case AArch64::SUBXri:
6674 case AArch64::SUBSXri:
6675 case AArch64::ADDv8i8:
6676 case AArch64::ADDv16i8:
6677 case AArch64::ADDv4i16:
6678 case AArch64::ADDv8i16:
6679 case AArch64::ADDv2i32:
6680 case AArch64::ADDv4i32:
6681 case AArch64::SUBv8i8:
6682 case AArch64::SUBv16i8:
6683 case AArch64::SUBv4i16:
6684 case AArch64::SUBv8i16:
6685 case AArch64::SUBv2i32:
6686 case AArch64::SUBv4i32:
6699 case AArch64::FADDHrr:
6700 case AArch64::FADDSrr:
6701 case AArch64::FADDDrr:
6702 case AArch64::FADDv4f16:
6703 case AArch64::FADDv8f16:
6704 case AArch64::FADDv2f32:
6705 case AArch64::FADDv2f64:
6706 case AArch64::FADDv4f32:
6707 case AArch64::FSUBHrr:
6708 case AArch64::FSUBSrr:
6709 case AArch64::FSUBDrr:
6710 case AArch64::FSUBv4f16:
6711 case AArch64::FSUBv8f16:
6712 case AArch64::FSUBv2f32:
6713 case AArch64::FSUBv2f64:
6714 case AArch64::FSUBv4f32:
6733 unsigned CombineOpc,
unsigned ZeroReg = 0,
6734 bool CheckZeroReg =
false) {
6741 if (!
MI ||
MI->getParent() != &
MBB ||
MI->getOpcode() != CombineOpc)
6744 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
6748 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
6749 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
6750 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
6752 if (
MI->getOperand(3).getReg() != ZeroReg)
6757 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
6766 unsigned MulOpc,
unsigned ZeroReg) {
6781bool AArch64InstrInfo::isAssociativeAndCommutative(
const MachineInstr &Inst,
6782 bool Invert)
const {
6788 case AArch64::FADDHrr:
6789 case AArch64::FADDSrr:
6790 case AArch64::FADDDrr:
6791 case AArch64::FMULHrr:
6792 case AArch64::FMULSrr:
6793 case AArch64::FMULDrr:
6794 case AArch64::FMULX16:
6795 case AArch64::FMULX32:
6796 case AArch64::FMULX64:
6798 case AArch64::FADDv4f16:
6799 case AArch64::FADDv8f16:
6800 case AArch64::FADDv2f32:
6801 case AArch64::FADDv4f32:
6802 case AArch64::FADDv2f64:
6803 case AArch64::FMULv4f16:
6804 case AArch64::FMULv8f16:
6805 case AArch64::FMULv2f32:
6806 case AArch64::FMULv4f32:
6807 case AArch64::FMULv2f64:
6808 case AArch64::FMULXv4f16:
6809 case AArch64::FMULXv8f16:
6810 case AArch64::FMULXv2f32:
6811 case AArch64::FMULXv4f32:
6812 case AArch64::FMULXv2f64:
6816 case AArch64::FADD_ZZZ_H:
6817 case AArch64::FADD_ZZZ_S:
6818 case AArch64::FADD_ZZZ_D:
6819 case AArch64::FMUL_ZZZ_H:
6820 case AArch64::FMUL_ZZZ_S:
6821 case AArch64::FMUL_ZZZ_D:
6832 case AArch64::ADDWrr:
6833 case AArch64::ADDXrr:
6834 case AArch64::ANDWrr:
6835 case AArch64::ANDXrr:
6836 case AArch64::ORRWrr:
6837 case AArch64::ORRXrr:
6838 case AArch64::EORWrr:
6839 case AArch64::EORXrr:
6840 case AArch64::EONWrr:
6841 case AArch64::EONXrr:
6845 case AArch64::ADDv8i8:
6846 case AArch64::ADDv16i8:
6847 case AArch64::ADDv4i16:
6848 case AArch64::ADDv8i16:
6849 case AArch64::ADDv2i32:
6850 case AArch64::ADDv4i32:
6851 case AArch64::ADDv1i64:
6852 case AArch64::ADDv2i64:
6853 case AArch64::MULv8i8:
6854 case AArch64::MULv16i8:
6855 case AArch64::MULv4i16:
6856 case AArch64::MULv8i16:
6857 case AArch64::MULv2i32:
6858 case AArch64::MULv4i32:
6859 case AArch64::ANDv8i8:
6860 case AArch64::ANDv16i8:
6861 case AArch64::ORRv8i8:
6862 case AArch64::ORRv16i8:
6863 case AArch64::EORv8i8:
6864 case AArch64::EORv16i8:
6866 case AArch64::ADD_ZZZ_B:
6867 case AArch64::ADD_ZZZ_H:
6868 case AArch64::ADD_ZZZ_S:
6869 case AArch64::ADD_ZZZ_D:
6870 case AArch64::MUL_ZZZ_B:
6871 case AArch64::MUL_ZZZ_H:
6872 case AArch64::MUL_ZZZ_S:
6873 case AArch64::MUL_ZZZ_D:
6874 case AArch64::AND_ZZZ:
6875 case AArch64::ORR_ZZZ:
6876 case AArch64::EOR_ZZZ:
6907 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6915 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6927 case AArch64::ADDWrr:
6929 "ADDWrr does not have register operands");
6930 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6931 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6933 case AArch64::ADDXrr:
6934 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6935 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6937 case AArch64::SUBWrr:
6938 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6939 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6941 case AArch64::SUBXrr:
6942 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6943 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6945 case AArch64::ADDWri:
6946 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6948 case AArch64::ADDXri:
6949 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6951 case AArch64::SUBWri:
6952 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6954 case AArch64::SUBXri:
6955 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6957 case AArch64::ADDv8i8:
6958 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6959 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6961 case AArch64::ADDv16i8:
6962 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6963 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6965 case AArch64::ADDv4i16:
6966 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6967 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6968 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6969 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6971 case AArch64::ADDv8i16:
6972 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6973 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6974 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6975 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6977 case AArch64::ADDv2i32:
6978 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6979 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6980 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6981 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6983 case AArch64::ADDv4i32:
6984 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6985 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6986 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6987 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6989 case AArch64::SUBv8i8:
6990 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6991 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6993 case AArch64::SUBv16i8:
6994 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6995 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6997 case AArch64::SUBv4i16:
6998 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6999 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7000 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7001 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7003 case AArch64::SUBv8i16:
7004 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7005 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7006 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7007 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7009 case AArch64::SUBv2i32:
7010 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7011 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7012 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7013 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7015 case AArch64::SUBv4i32:
7016 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7017 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7018 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7019 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7025bool AArch64InstrInfo::isAccumulationOpcode(
unsigned Opcode)
const {
7029 case AArch64::UABALB_ZZZ_D:
7030 case AArch64::UABALB_ZZZ_H:
7031 case AArch64::UABALB_ZZZ_S:
7032 case AArch64::UABALT_ZZZ_D:
7033 case AArch64::UABALT_ZZZ_H:
7034 case AArch64::UABALT_ZZZ_S:
7035 case AArch64::SABALB_ZZZ_D:
7036 case AArch64::SABALB_ZZZ_S:
7037 case AArch64::SABALB_ZZZ_H:
7038 case AArch64::SABALT_ZZZ_D:
7039 case AArch64::SABALT_ZZZ_S:
7040 case AArch64::SABALT_ZZZ_H:
7041 case AArch64::UABALv16i8_v8i16:
7042 case AArch64::UABALv2i32_v2i64:
7043 case AArch64::UABALv4i16_v4i32:
7044 case AArch64::UABALv4i32_v2i64:
7045 case AArch64::UABALv8i16_v4i32:
7046 case AArch64::UABALv8i8_v8i16:
7047 case AArch64::UABAv16i8:
7048 case AArch64::UABAv2i32:
7049 case AArch64::UABAv4i16:
7050 case AArch64::UABAv4i32:
7051 case AArch64::UABAv8i16:
7052 case AArch64::UABAv8i8:
7053 case AArch64::SABALv16i8_v8i16:
7054 case AArch64::SABALv2i32_v2i64:
7055 case AArch64::SABALv4i16_v4i32:
7056 case AArch64::SABALv4i32_v2i64:
7057 case AArch64::SABALv8i16_v4i32:
7058 case AArch64::SABALv8i8_v8i16:
7059 case AArch64::SABAv16i8:
7060 case AArch64::SABAv2i32:
7061 case AArch64::SABAv4i16:
7062 case AArch64::SABAv4i32:
7063 case AArch64::SABAv8i16:
7064 case AArch64::SABAv8i8:
7071unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7072 unsigned AccumulationOpcode)
const {
7073 switch (AccumulationOpcode) {
7076 case AArch64::UABALB_ZZZ_D:
7077 return AArch64::UABDLB_ZZZ_D;
7078 case AArch64::UABALB_ZZZ_H:
7079 return AArch64::UABDLB_ZZZ_H;
7080 case AArch64::UABALB_ZZZ_S:
7081 return AArch64::UABDLB_ZZZ_S;
7082 case AArch64::UABALT_ZZZ_D:
7083 return AArch64::UABDLT_ZZZ_D;
7084 case AArch64::UABALT_ZZZ_H:
7085 return AArch64::UABDLT_ZZZ_H;
7086 case AArch64::UABALT_ZZZ_S:
7087 return AArch64::UABDLT_ZZZ_S;
7088 case AArch64::UABALv16i8_v8i16:
7089 return AArch64::UABDLv16i8_v8i16;
7090 case AArch64::UABALv2i32_v2i64:
7091 return AArch64::UABDLv2i32_v2i64;
7092 case AArch64::UABALv4i16_v4i32:
7093 return AArch64::UABDLv4i16_v4i32;
7094 case AArch64::UABALv4i32_v2i64:
7095 return AArch64::UABDLv4i32_v2i64;
7096 case AArch64::UABALv8i16_v4i32:
7097 return AArch64::UABDLv8i16_v4i32;
7098 case AArch64::UABALv8i8_v8i16:
7099 return AArch64::UABDLv8i8_v8i16;
7100 case AArch64::UABAv16i8:
7101 return AArch64::UABDv16i8;
7102 case AArch64::UABAv2i32:
7103 return AArch64::UABDv2i32;
7104 case AArch64::UABAv4i16:
7105 return AArch64::UABDv4i16;
7106 case AArch64::UABAv4i32:
7107 return AArch64::UABDv4i32;
7108 case AArch64::UABAv8i16:
7109 return AArch64::UABDv8i16;
7110 case AArch64::UABAv8i8:
7111 return AArch64::UABDv8i8;
7112 case AArch64::SABALB_ZZZ_D:
7113 return AArch64::SABDLB_ZZZ_D;
7114 case AArch64::SABALB_ZZZ_S:
7115 return AArch64::SABDLB_ZZZ_S;
7116 case AArch64::SABALB_ZZZ_H:
7117 return AArch64::SABDLB_ZZZ_H;
7118 case AArch64::SABALT_ZZZ_D:
7119 return AArch64::SABDLT_ZZZ_D;
7120 case AArch64::SABALT_ZZZ_S:
7121 return AArch64::SABDLT_ZZZ_S;
7122 case AArch64::SABALT_ZZZ_H:
7123 return AArch64::SABDLT_ZZZ_H;
7124 case AArch64::SABALv16i8_v8i16:
7125 return AArch64::SABDLv16i8_v8i16;
7126 case AArch64::SABALv2i32_v2i64:
7127 return AArch64::SABDLv2i32_v2i64;
7128 case AArch64::SABALv4i16_v4i32:
7129 return AArch64::SABDLv4i16_v4i32;
7130 case AArch64::SABALv4i32_v2i64:
7131 return AArch64::SABDLv4i32_v2i64;
7132 case AArch64::SABALv8i16_v4i32:
7133 return AArch64::SABDLv8i16_v4i32;
7134 case AArch64::SABALv8i8_v8i16:
7135 return AArch64::SABDLv8i8_v8i16;
7136 case AArch64::SABAv16i8:
7137 return AArch64::SABDv16i8;
7138 case AArch64::SABAv2i32:
7139 return AArch64::SABAv2i32;
7140 case AArch64::SABAv4i16:
7141 return AArch64::SABDv4i16;
7142 case AArch64::SABAv4i32:
7143 return AArch64::SABDv4i32;
7144 case AArch64::SABAv8i16:
7145 return AArch64::SABDv8i16;
7146 case AArch64::SABAv8i8:
7147 return AArch64::SABDv8i8;
7163 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
7175 assert(
false &&
"Unsupported FP instruction in combiner\n");
7177 case AArch64::FADDHrr:
7179 "FADDHrr does not have register operands");
7181 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
7182 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
7184 case AArch64::FADDSrr:
7186 "FADDSrr does not have register operands");
7188 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
7189 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
7191 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
7192 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
7194 case AArch64::FADDDrr:
7195 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
7196 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
7198 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
7199 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
7201 case AArch64::FADDv4f16:
7202 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
7203 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
7205 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
7206 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
7208 case AArch64::FADDv8f16:
7209 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
7210 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
7212 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
7213 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
7215 case AArch64::FADDv2f32:
7216 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
7217 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
7219 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
7220 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
7222 case AArch64::FADDv2f64:
7223 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
7224 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
7226 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
7227 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
7229 case AArch64::FADDv4f32:
7230 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
7231 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
7233 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
7234 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
7236 case AArch64::FSUBHrr:
7237 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
7238 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
7239 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
7241 case AArch64::FSUBSrr:
7242 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
7244 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
7245 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
7247 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
7249 case AArch64::FSUBDrr:
7250 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
7252 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
7253 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
7255 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
7257 case AArch64::FSUBv4f16:
7258 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
7259 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
7261 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
7262 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
7264 case AArch64::FSUBv8f16:
7265 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
7266 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
7268 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
7269 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
7271 case AArch64::FSUBv2f32:
7272 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
7273 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
7275 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
7276 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
7278 case AArch64::FSUBv2f64:
7279 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
7280 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
7282 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
7283 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
7285 case AArch64::FSUBv4f32:
7286 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
7287 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
7289 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
7290 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
7301 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
7308 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
7309 MI->getOperand(1).getReg().isVirtual())
7310 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
7311 if (
MI &&
MI->getOpcode() == Opcode) {
7323 case AArch64::FMULv2f32:
7324 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
7325 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
7327 case AArch64::FMULv2f64:
7328 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
7329 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
7331 case AArch64::FMULv4f16:
7332 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
7333 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
7335 case AArch64::FMULv4f32:
7336 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
7337 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
7339 case AArch64::FMULv8f16:
7340 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
7341 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
7354 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
7357 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
7358 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
7372 case AArch64::FNEGDr:
7374 case AArch64::FNEGSr:
7506 case AArch64::SUBWrr:
7507 case AArch64::SUBSWrr:
7508 case AArch64::SUBXrr:
7509 case AArch64::SUBSXrr:
7554 unsigned LoadLaneOpCode,
unsigned NumLanes) {
7577 while (!RemainingLanes.
empty() && CurrInstr &&
7578 CurrInstr->getOpcode() == LoadLaneOpCode &&
7579 MRI.hasOneNonDBGUse(CurrInstr->getOperand(0).getReg()) &&
7580 CurrInstr->getNumOperands() == 4) {
7581 RemainingLanes.
erase(CurrInstr->getOperand(2).getImm());
7583 CurrInstr =
MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
7587 if (!RemainingLanes.
empty())
7591 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
7595 auto Lane0LoadReg = CurrInstr->getOperand(2).getReg();
7596 unsigned SingleLaneSizeInBits = 128 / NumLanes;
7597 if (
TRI->getRegSizeInBits(Lane0LoadReg,
MRI) != SingleLaneSizeInBits)
7601 if (!
MRI.hasOneNonDBGUse(Lane0LoadReg))
7604 LoadInstrs.
push_back(
MRI.getUniqueVRegDef(Lane0LoadReg));
7613 RemainingLoadInstrs.
insert(LoadInstrs.
begin(), LoadInstrs.
end());
7616 for (; MBBItr !=
MBB->begin() && RemainingSteps > 0 &&
7617 !RemainingLoadInstrs.
empty();
7618 --MBBItr, --RemainingSteps) {
7622 RemainingLoadInstrs.
erase(&CurrInstr);
7632 if (RemainingSteps == 0 && !RemainingLoadInstrs.
empty())
7658 case AArch64::LD1i32:
7660 case AArch64::LD1i16:
7662 case AArch64::LD1i8:
7678 unsigned Pattern,
unsigned NumLanes) {
7686 for (
unsigned i = 0; i < NumLanes - 1; ++i) {
7694 return A->getOperand(2).getImm() >
B->getOperand(2).getImm();
7699 MRI.getUniqueVRegDef(SubregToReg->getOperand(2).getReg()));
7700 auto LoadToLaneInstrsAscending =
llvm::reverse(LoadToLaneInstrs);
7706 auto CreateLD1Instruction = [&](
MachineInstr *OriginalInstr,
7707 Register SrcRegister,
unsigned Lane,
7709 bool OffsetRegisterKillState) {
7710 auto NewRegister =
MRI.createVirtualRegister(FPR128RegClass);
7717 InstrIdxForVirtReg.
insert(std::make_pair(NewRegister, InsInstrs.
size()));
7718 InsInstrs.
push_back(LoadIndexIntoRegister);
7724 auto CreateLDRInstruction = [&](
unsigned NumLanes,
Register DestReg,
7730 Opcode = AArch64::LDRSui;
7733 Opcode = AArch64::LDRHui;
7736 Opcode = AArch64::LDRBui;
7740 "Got unsupported number of lanes in machine-combiner gather pattern");
7749 auto LanesToLoadToReg0 =
7751 LoadToLaneInstrsAscending.begin() + NumLanes / 2);
7752 Register PrevReg = SubregToReg->getOperand(0).getReg();
7754 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
7755 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
7756 OffsetRegOperand.
getReg(),
7757 OffsetRegOperand.
isKill());
7764 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
7766 *std::next(LoadToLaneInstrsAscending.begin(), NumLanes / 2);
7767 Register DestRegForMiddleIndex =
MRI.createVirtualRegister(
7773 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
7774 OriginalSplitToLoadOffsetOperand.
getReg(),
7775 OriginalSplitToLoadOffsetOperand.
isKill());
7777 InstrIdxForVirtReg.
insert(
7778 std::make_pair(DestRegForMiddleIndex, InsInstrs.
size()));
7779 InsInstrs.
push_back(MiddleIndexLoadInstr);
7783 Register DestRegForSubregToReg =
MRI.createVirtualRegister(FPR128RegClass);
7784 unsigned SubregType;
7787 SubregType = AArch64::ssub;
7790 SubregType = AArch64::hsub;
7793 SubregType = AArch64::bsub;
7797 "Got invalid NumLanes for machine-combiner gather pattern");
7800 auto SubRegToRegInstr =
7802 DestRegForSubregToReg)
7806 InstrIdxForVirtReg.
insert(
7807 std::make_pair(DestRegForSubregToReg, InsInstrs.
size()));
7811 auto LanesToLoadToReg1 =
7813 LoadToLaneInstrsAscending.end());
7814 PrevReg = SubRegToRegInstr->getOperand(0).getReg();
7816 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
7817 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
7818 OffsetRegOperand.
getReg(),
7819 OffsetRegOperand.
isKill());
7822 if (Index == NumLanes / 2 - 2) {
7857bool AArch64InstrInfo::getMachineCombinerPatterns(
7859 bool DoRegPressureReduce)
const {
7880 DoRegPressureReduce);
7909 const Register *ReplacedAddend =
nullptr) {
7910 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7912 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
7915 Register SrcReg0 = MUL->getOperand(1).getReg();
7916 bool Src0IsKill = MUL->getOperand(1).isKill();
7917 Register SrcReg1 = MUL->getOperand(2).getReg();
7918 bool Src1IsKill = MUL->getOperand(2).isKill();
7922 if (ReplacedAddend) {
7924 SrcReg2 = *ReplacedAddend;
7932 MRI.constrainRegClass(ResultReg, RC);
7934 MRI.constrainRegClass(SrcReg0, RC);
7936 MRI.constrainRegClass(SrcReg1, RC);
7938 MRI.constrainRegClass(SrcReg2, RC);
7951 .
addImm(MUL->getOperand(3).getImm());
7958 assert(
false &&
"Invalid FMA instruction kind \n");
7972 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
7973 Opc = AArch64::FNMADDSrrr;
7974 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
7975 Opc = AArch64::FNMADDDrrr;
7987 MRI.constrainRegClass(ResultReg, RC);
7989 MRI.constrainRegClass(SrcReg0, RC);
7991 MRI.constrainRegClass(SrcReg1, RC);
7993 MRI.constrainRegClass(SrcReg2, RC);
8009 unsigned IdxDupOp,
unsigned MulOpc,
8011 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8012 "Invalid index of FMUL operand");
8020 if (Dup->
getOpcode() == TargetOpcode::COPY)
8024 MRI.clearKillFlags(DupSrcReg);
8025 MRI.constrainRegClass(DupSrcReg, RC);
8029 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8070 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8085 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8112 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8140 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
8142 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8146 Register SrcReg0 = MUL->getOperand(1).getReg();
8147 bool Src0IsKill = MUL->getOperand(1).isKill();
8148 Register SrcReg1 = MUL->getOperand(2).getReg();
8149 bool Src1IsKill = MUL->getOperand(2).isKill();
8152 MRI.constrainRegClass(ResultReg, RC);
8154 MRI.constrainRegClass(SrcReg0, RC);
8156 MRI.constrainRegClass(SrcReg1, RC);
8158 MRI.constrainRegClass(VR, RC);
8179 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
8180 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
8194 if (Opcode == AArch64::SUBSWrr)
8195 Opcode = AArch64::SUBWrr;
8196 else if (Opcode == AArch64::SUBSXrr)
8197 Opcode = AArch64::SUBXrr;
8199 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
8200 "Unexpected instruction opcode.");
8217 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8224unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
8225 unsigned int AccumulatorOpCode)
const {
8226 switch (AccumulatorOpCode) {
8227 case AArch64::UABALB_ZZZ_D:
8228 case AArch64::SABALB_ZZZ_D:
8229 case AArch64::UABALT_ZZZ_D:
8230 case AArch64::SABALT_ZZZ_D:
8231 return AArch64::ADD_ZZZ_D;
8232 case AArch64::UABALB_ZZZ_H:
8233 case AArch64::SABALB_ZZZ_H:
8234 case AArch64::UABALT_ZZZ_H:
8235 case AArch64::SABALT_ZZZ_H:
8236 return AArch64::ADD_ZZZ_H;
8237 case AArch64::UABALB_ZZZ_S:
8238 case AArch64::SABALB_ZZZ_S:
8239 case AArch64::UABALT_ZZZ_S:
8240 case AArch64::SABALT_ZZZ_S:
8241 return AArch64::ADD_ZZZ_S;
8242 case AArch64::UABALv16i8_v8i16:
8243 case AArch64::SABALv8i8_v8i16:
8244 case AArch64::SABAv8i16:
8245 case AArch64::UABAv8i16:
8246 return AArch64::ADDv8i16;
8247 case AArch64::SABALv2i32_v2i64:
8248 case AArch64::UABALv2i32_v2i64:
8249 case AArch64::SABALv4i32_v2i64:
8250 return AArch64::ADDv2i64;
8251 case AArch64::UABALv4i16_v4i32:
8252 case AArch64::SABALv4i16_v4i32:
8253 case AArch64::SABALv8i16_v4i32:
8254 case AArch64::SABAv4i32:
8255 case AArch64::UABAv4i32:
8256 return AArch64::ADDv4i32;
8257 case AArch64::UABALv4i32_v2i64:
8258 return AArch64::ADDv2i64;
8259 case AArch64::UABALv8i16_v4i32:
8260 return AArch64::ADDv4i32;
8261 case AArch64::UABALv8i8_v8i16:
8262 case AArch64::SABALv16i8_v8i16:
8263 return AArch64::ADDv8i16;
8264 case AArch64::UABAv16i8:
8265 case AArch64::SABAv16i8:
8266 return AArch64::ADDv16i8;
8267 case AArch64::UABAv4i16:
8268 case AArch64::SABAv4i16:
8269 return AArch64::ADDv4i16;
8270 case AArch64::UABAv2i32:
8271 case AArch64::SABAv2i32:
8272 return AArch64::ADDv2i32;
8273 case AArch64::UABAv8i8:
8274 case AArch64::SABAv8i8:
8275 return AArch64::ADDv8i8;
8284void AArch64InstrInfo::genAlternativeCodeSequence(
8294 MachineInstr *
MUL =
nullptr;
8295 const TargetRegisterClass *RC;
8301 DelInstrs, InstrIdxForVirtReg);
8307 InstrIdxForVirtReg);
8313 InstrIdxForVirtReg);
8322 Opc = AArch64::MADDWrrr;
8323 RC = &AArch64::GPR32RegClass;
8325 Opc = AArch64::MADDXrrr;
8326 RC = &AArch64::GPR64RegClass;
8337 Opc = AArch64::MADDWrrr;
8338 RC = &AArch64::GPR32RegClass;
8340 Opc = AArch64::MADDXrrr;
8341 RC = &AArch64::GPR64RegClass;
8354 const TargetRegisterClass *RC;
8355 unsigned BitSize, MovImm;
8358 MovImm = AArch64::MOVi32imm;
8359 RC = &AArch64::GPR32spRegClass;
8361 Opc = AArch64::MADDWrrr;
8362 RC = &AArch64::GPR32RegClass;
8364 MovImm = AArch64::MOVi64imm;
8365 RC = &AArch64::GPR64spRegClass;
8367 Opc = AArch64::MADDXrrr;
8368 RC = &AArch64::GPR64RegClass;
8379 uint64_t UImm =
SignExtend64(IsSub ? -Imm : Imm, BitSize);
8383 if (Insn.
size() != 1)
8385 MachineInstrBuilder MIB1 =
8386 BuildMI(MF, MIMetadata(Root),
TII->get(MovImm), NewVR)
8387 .
addImm(IsSub ? -Imm : Imm);
8389 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8400 const TargetRegisterClass *SubRC;
8401 unsigned SubOpc, ZeroReg;
8403 SubOpc = AArch64::SUBWrr;
8404 SubRC = &AArch64::GPR32spRegClass;
8405 ZeroReg = AArch64::WZR;
8406 Opc = AArch64::MADDWrrr;
8407 RC = &AArch64::GPR32RegClass;
8409 SubOpc = AArch64::SUBXrr;
8410 SubRC = &AArch64::GPR64spRegClass;
8411 ZeroReg = AArch64::XZR;
8412 Opc = AArch64::MADDXrrr;
8413 RC = &AArch64::GPR64RegClass;
8415 Register NewVR =
MRI.createVirtualRegister(SubRC);
8417 MachineInstrBuilder MIB1 =
8418 BuildMI(MF, MIMetadata(Root),
TII->get(SubOpc), NewVR)
8422 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8433 Opc = AArch64::MSUBWrrr;
8434 RC = &AArch64::GPR32RegClass;
8436 Opc = AArch64::MSUBXrrr;
8437 RC = &AArch64::GPR64RegClass;
8442 Opc = AArch64::MLAv8i8;
8443 RC = &AArch64::FPR64RegClass;
8447 Opc = AArch64::MLAv8i8;
8448 RC = &AArch64::FPR64RegClass;
8452 Opc = AArch64::MLAv16i8;
8453 RC = &AArch64::FPR128RegClass;
8457 Opc = AArch64::MLAv16i8;
8458 RC = &AArch64::FPR128RegClass;
8462 Opc = AArch64::MLAv4i16;
8463 RC = &AArch64::FPR64RegClass;
8467 Opc = AArch64::MLAv4i16;
8468 RC = &AArch64::FPR64RegClass;
8472 Opc = AArch64::MLAv8i16;
8473 RC = &AArch64::FPR128RegClass;
8477 Opc = AArch64::MLAv8i16;
8478 RC = &AArch64::FPR128RegClass;
8482 Opc = AArch64::MLAv2i32;
8483 RC = &AArch64::FPR64RegClass;
8487 Opc = AArch64::MLAv2i32;
8488 RC = &AArch64::FPR64RegClass;
8492 Opc = AArch64::MLAv4i32;
8493 RC = &AArch64::FPR128RegClass;
8497 Opc = AArch64::MLAv4i32;
8498 RC = &AArch64::FPR128RegClass;
8503 Opc = AArch64::MLAv8i8;
8504 RC = &AArch64::FPR64RegClass;
8506 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i8,
8510 Opc = AArch64::MLSv8i8;
8511 RC = &AArch64::FPR64RegClass;
8515 Opc = AArch64::MLAv16i8;
8516 RC = &AArch64::FPR128RegClass;
8518 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv16i8,
8522 Opc = AArch64::MLSv16i8;
8523 RC = &AArch64::FPR128RegClass;
8527 Opc = AArch64::MLAv4i16;
8528 RC = &AArch64::FPR64RegClass;
8530 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i16,
8534 Opc = AArch64::MLSv4i16;
8535 RC = &AArch64::FPR64RegClass;
8539 Opc = AArch64::MLAv8i16;
8540 RC = &AArch64::FPR128RegClass;
8542 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i16,
8546 Opc = AArch64::MLSv8i16;
8547 RC = &AArch64::FPR128RegClass;
8551 Opc = AArch64::MLAv2i32;
8552 RC = &AArch64::FPR64RegClass;
8554 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv2i32,
8558 Opc = AArch64::MLSv2i32;
8559 RC = &AArch64::FPR64RegClass;
8563 Opc = AArch64::MLAv4i32;
8564 RC = &AArch64::FPR128RegClass;
8566 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i32,
8570 Opc = AArch64::MLSv4i32;
8571 RC = &AArch64::FPR128RegClass;
8576 Opc = AArch64::MLAv4i16_indexed;
8577 RC = &AArch64::FPR64RegClass;
8581 Opc = AArch64::MLAv4i16_indexed;
8582 RC = &AArch64::FPR64RegClass;
8586 Opc = AArch64::MLAv8i16_indexed;
8587 RC = &AArch64::FPR128RegClass;
8591 Opc = AArch64::MLAv8i16_indexed;
8592 RC = &AArch64::FPR128RegClass;
8596 Opc = AArch64::MLAv2i32_indexed;
8597 RC = &AArch64::FPR64RegClass;
8601 Opc = AArch64::MLAv2i32_indexed;
8602 RC = &AArch64::FPR64RegClass;
8606 Opc = AArch64::MLAv4i32_indexed;
8607 RC = &AArch64::FPR128RegClass;
8611 Opc = AArch64::MLAv4i32_indexed;
8612 RC = &AArch64::FPR128RegClass;
8617 Opc = AArch64::MLAv4i16_indexed;
8618 RC = &AArch64::FPR64RegClass;
8620 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i16,
8624 Opc = AArch64::MLSv4i16_indexed;
8625 RC = &AArch64::FPR64RegClass;
8629 Opc = AArch64::MLAv8i16_indexed;
8630 RC = &AArch64::FPR128RegClass;
8632 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i16,
8636 Opc = AArch64::MLSv8i16_indexed;
8637 RC = &AArch64::FPR128RegClass;
8641 Opc = AArch64::MLAv2i32_indexed;
8642 RC = &AArch64::FPR64RegClass;
8644 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv2i32,
8648 Opc = AArch64::MLSv2i32_indexed;
8649 RC = &AArch64::FPR64RegClass;
8653 Opc = AArch64::MLAv4i32_indexed;
8654 RC = &AArch64::FPR128RegClass;
8656 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i32,
8660 Opc = AArch64::MLSv4i32_indexed;
8661 RC = &AArch64::FPR128RegClass;
8667 Opc = AArch64::FMADDHrrr;
8668 RC = &AArch64::FPR16RegClass;
8672 Opc = AArch64::FMADDSrrr;
8673 RC = &AArch64::FPR32RegClass;
8677 Opc = AArch64::FMADDDrrr;
8678 RC = &AArch64::FPR64RegClass;
8683 Opc = AArch64::FMADDHrrr;
8684 RC = &AArch64::FPR16RegClass;
8688 Opc = AArch64::FMADDSrrr;
8689 RC = &AArch64::FPR32RegClass;
8693 Opc = AArch64::FMADDDrrr;
8694 RC = &AArch64::FPR64RegClass;
8699 Opc = AArch64::FMLAv1i32_indexed;
8700 RC = &AArch64::FPR32RegClass;
8705 Opc = AArch64::FMLAv1i32_indexed;
8706 RC = &AArch64::FPR32RegClass;
8712 Opc = AArch64::FMLAv1i64_indexed;
8713 RC = &AArch64::FPR64RegClass;
8718 Opc = AArch64::FMLAv1i64_indexed;
8719 RC = &AArch64::FPR64RegClass;
8725 RC = &AArch64::FPR64RegClass;
8726 Opc = AArch64::FMLAv4i16_indexed;
8731 RC = &AArch64::FPR64RegClass;
8732 Opc = AArch64::FMLAv4f16;
8737 RC = &AArch64::FPR64RegClass;
8738 Opc = AArch64::FMLAv4i16_indexed;
8743 RC = &AArch64::FPR64RegClass;
8744 Opc = AArch64::FMLAv4f16;
8751 RC = &AArch64::FPR64RegClass;
8753 Opc = AArch64::FMLAv2i32_indexed;
8757 Opc = AArch64::FMLAv2f32;
8764 RC = &AArch64::FPR64RegClass;
8766 Opc = AArch64::FMLAv2i32_indexed;
8770 Opc = AArch64::FMLAv2f32;
8777 RC = &AArch64::FPR128RegClass;
8778 Opc = AArch64::FMLAv8i16_indexed;
8783 RC = &AArch64::FPR128RegClass;
8784 Opc = AArch64::FMLAv8f16;
8789 RC = &AArch64::FPR128RegClass;
8790 Opc = AArch64::FMLAv8i16_indexed;
8795 RC = &AArch64::FPR128RegClass;
8796 Opc = AArch64::FMLAv8f16;
8803 RC = &AArch64::FPR128RegClass;
8805 Opc = AArch64::FMLAv2i64_indexed;
8809 Opc = AArch64::FMLAv2f64;
8816 RC = &AArch64::FPR128RegClass;
8818 Opc = AArch64::FMLAv2i64_indexed;
8822 Opc = AArch64::FMLAv2f64;
8830 RC = &AArch64::FPR128RegClass;
8832 Opc = AArch64::FMLAv4i32_indexed;
8836 Opc = AArch64::FMLAv4f32;
8844 RC = &AArch64::FPR128RegClass;
8846 Opc = AArch64::FMLAv4i32_indexed;
8850 Opc = AArch64::FMLAv4f32;
8857 Opc = AArch64::FNMSUBHrrr;
8858 RC = &AArch64::FPR16RegClass;
8862 Opc = AArch64::FNMSUBSrrr;
8863 RC = &AArch64::FPR32RegClass;
8867 Opc = AArch64::FNMSUBDrrr;
8868 RC = &AArch64::FPR64RegClass;
8873 Opc = AArch64::FNMADDHrrr;
8874 RC = &AArch64::FPR16RegClass;
8878 Opc = AArch64::FNMADDSrrr;
8879 RC = &AArch64::FPR32RegClass;
8883 Opc = AArch64::FNMADDDrrr;
8884 RC = &AArch64::FPR64RegClass;
8889 Opc = AArch64::FMSUBHrrr;
8890 RC = &AArch64::FPR16RegClass;
8894 Opc = AArch64::FMSUBSrrr;
8895 RC = &AArch64::FPR32RegClass;
8899 Opc = AArch64::FMSUBDrrr;
8900 RC = &AArch64::FPR64RegClass;
8905 Opc = AArch64::FMLSv1i32_indexed;
8906 RC = &AArch64::FPR32RegClass;
8912 Opc = AArch64::FMLSv1i64_indexed;
8913 RC = &AArch64::FPR64RegClass;
8920 RC = &AArch64::FPR64RegClass;
8922 MachineInstrBuilder MIB1 =
8923 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv4f16), NewVR)
8926 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8928 Opc = AArch64::FMLAv4f16;
8932 Opc = AArch64::FMLAv4i16_indexed;
8939 RC = &AArch64::FPR64RegClass;
8940 Opc = AArch64::FMLSv4f16;
8945 RC = &AArch64::FPR64RegClass;
8946 Opc = AArch64::FMLSv4i16_indexed;
8953 RC = &AArch64::FPR64RegClass;
8955 Opc = AArch64::FMLSv2i32_indexed;
8959 Opc = AArch64::FMLSv2f32;
8967 RC = &AArch64::FPR128RegClass;
8969 MachineInstrBuilder MIB1 =
8970 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv8f16), NewVR)
8973 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8975 Opc = AArch64::FMLAv8f16;
8979 Opc = AArch64::FMLAv8i16_indexed;
8986 RC = &AArch64::FPR128RegClass;
8987 Opc = AArch64::FMLSv8f16;
8992 RC = &AArch64::FPR128RegClass;
8993 Opc = AArch64::FMLSv8i16_indexed;
9000 RC = &AArch64::FPR128RegClass;
9002 Opc = AArch64::FMLSv2i64_indexed;
9006 Opc = AArch64::FMLSv2f64;
9014 RC = &AArch64::FPR128RegClass;
9016 Opc = AArch64::FMLSv4i32_indexed;
9020 Opc = AArch64::FMLSv4f32;
9027 RC = &AArch64::FPR64RegClass;
9029 MachineInstrBuilder MIB1 =
9030 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv2f32), NewVR)
9033 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9035 Opc = AArch64::FMLAv2i32_indexed;
9039 Opc = AArch64::FMLAv2f32;
9047 RC = &AArch64::FPR128RegClass;
9049 MachineInstrBuilder MIB1 =
9050 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv4f32), NewVR)
9053 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9055 Opc = AArch64::FMLAv4i32_indexed;
9059 Opc = AArch64::FMLAv4f32;
9067 RC = &AArch64::FPR128RegClass;
9069 MachineInstrBuilder MIB1 =
9070 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv2f64), NewVR)
9073 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9075 Opc = AArch64::FMLAv2i64_indexed;
9079 Opc = AArch64::FMLAv2f64;
9091 &AArch64::FPR128RegClass,
MRI);
9100 &AArch64::FPR128RegClass,
MRI);
9109 &AArch64::FPR128_loRegClass,
MRI);
9118 &AArch64::FPR128RegClass,
MRI);
9127 &AArch64::FPR128_loRegClass,
MRI);
9161 for (
auto *
MI : InsInstrs)
9162 MI->setFlags(Flags);
9203 bool IsNegativeBranch =
false;
9204 bool IsTestAndBranch =
false;
9205 unsigned TargetBBInMI = 0;
9206 switch (
MI.getOpcode()) {
9210 case AArch64::CBWPri:
9211 case AArch64::CBXPri:
9212 case AArch64::CBWPrr:
9213 case AArch64::CBXPrr:
9219 case AArch64::CBNZW:
9220 case AArch64::CBNZX:
9222 IsNegativeBranch =
true;
9227 IsTestAndBranch =
true;
9229 case AArch64::TBNZW:
9230 case AArch64::TBNZX:
9232 IsNegativeBranch =
true;
9233 IsTestAndBranch =
true;
9239 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
9243 assert(
MI.getParent() &&
"Incomplete machine instruction\n");
9254 while (
DefMI->isCopy()) {
9256 if (!
MRI->hasOneNonDBGUse(CopyVReg))
9258 if (!
MRI->hasOneDef(CopyVReg))
9263 switch (
DefMI->getOpcode()) {
9267 case AArch64::ANDWri:
9268 case AArch64::ANDXri: {
9269 if (IsTestAndBranch)
9273 if (!
MRI->hasOneNonDBGUse(VReg))
9276 bool Is32Bit = (
DefMI->getOpcode() == AArch64::ANDWri);
9278 DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
9287 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
9293 unsigned Opc = (Imm < 32)
9294 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
9295 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
9308 if (!Is32Bit && Imm < 32)
9310 MI.eraseFromParent();
9314 case AArch64::CSINCWr:
9315 case AArch64::CSINCXr: {
9316 if (!(
DefMI->getOperand(1).getReg() == AArch64::WZR &&
9317 DefMI->getOperand(2).getReg() == AArch64::WZR) &&
9318 !(
DefMI->getOperand(1).getReg() == AArch64::XZR &&
9319 DefMI->getOperand(2).getReg() == AArch64::XZR))
9322 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
9335 if (IsNegativeBranch)
9338 MI.eraseFromParent();
9344std::pair<unsigned, unsigned>
9345AArch64InstrInfo::decomposeMachineOperandsTargetFlags(
unsigned TF)
const {
9347 return std::make_pair(TF & Mask, TF & ~Mask);
9351AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags()
const {
9354 static const std::pair<unsigned, const char *> TargetFlags[] = {
9355 {MO_PAGE,
"aarch64-page"}, {
MO_PAGEOFF,
"aarch64-pageoff"},
9356 {
MO_G3,
"aarch64-g3"}, {
MO_G2,
"aarch64-g2"},
9357 {
MO_G1,
"aarch64-g1"}, {
MO_G0,
"aarch64-g0"},
9363AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags()
const {
9364 using namespace AArch64II;
9366 static const std::pair<unsigned, const char *> TargetFlags[] = {
9369 {
MO_NC,
"aarch64-nc"},
9370 {
MO_S,
"aarch64-s"},
9381AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags()
const {
9382 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
9484 MachineFunction *MF =
C.getMF();
9486 const AArch64RegisterInfo *ARI =
9487 static_cast<const AArch64RegisterInfo *
>(&
TRI);
9490 for (
unsigned Reg : AArch64::GPR64RegClass) {
9492 Reg != AArch64::LR &&
9493 Reg != AArch64::X16 &&
9494 Reg != AArch64::X17 &&
9495 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
9496 C.isAvailableInsideSeq(
Reg,
TRI))
9527 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
9530std::optional<std::unique_ptr<outliner::OutlinedFunction>>
9531AArch64InstrInfo::getOutliningCandidateInfo(
9533 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
9534 unsigned MinRepeats)
const {
9535 unsigned SequenceSize = 0;
9536 for (
auto &
MI : RepeatedSequenceLocs[0])
9539 unsigned NumBytesToCreateFrame = 0;
9549 if (std::adjacent_find(
9550 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
9551 [](
const outliner::Candidate &a,
const outliner::Candidate &b) {
9554 if (outliningCandidatesSigningScopeConsensus(a, b) &&
9555 outliningCandidatesSigningKeyConsensus(a, b) &&
9556 outliningCandidatesV8_3OpsConsensus(a, b)) {
9560 }) != RepeatedSequenceLocs.end()) {
9561 return std::nullopt;
9578 unsigned NumBytesToCheckLRInTCEpilogue = 0;
9579 if (RepeatedSequenceLocs[0]
9581 ->getInfo<AArch64FunctionInfo>()
9582 ->shouldSignReturnAddress(
true)) {
9584 NumBytesToCreateFrame += 8;
9587 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
9588 *RepeatedSequenceLocs[0].getMF());
9589 NumBytesToCheckLRInTCEpilogue =
9593 if (isTailCallReturnInst(RepeatedSequenceLocs[0].
back()))
9594 SequenceSize += NumBytesToCheckLRInTCEpilogue;
9602 for (
auto &
MI :
C) {
9603 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
9604 switch (
MI.getOpcode()) {
9605 case AArch64::ADDXri:
9606 case AArch64::ADDWri:
9607 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
9609 "Expected operand to be immediate");
9611 "Expected operand to be a register");
9615 if (
MI.getOperand(1).getReg() == AArch64::SP)
9616 SPValue +=
MI.getOperand(2).getImm();
9620 case AArch64::SUBXri:
9621 case AArch64::SUBWri:
9622 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
9624 "Expected operand to be immediate");
9626 "Expected operand to be a register");
9630 if (
MI.getOperand(1).getReg() == AArch64::SP)
9631 SPValue -=
MI.getOperand(2).getImm();
9648 if (RepeatedSequenceLocs.size() < MinRepeats)
9649 return std::nullopt;
9653 unsigned FlagsSetInAll = 0xF;
9657 FlagsSetInAll &=
C.Flags;
9659 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
9662 auto SetCandidateCallInfo =
9663 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
9665 C.setCallInfo(CallID, NumBytesForCall);
9669 NumBytesToCreateFrame += 4;
9677 unsigned CFICount = 0;
9678 for (
auto &
I : RepeatedSequenceLocs[0]) {
9679 if (
I.isCFIInstruction())
9689 std::vector<MCCFIInstruction> CFIInstructions =
9690 C.getMF()->getFrameInstructions();
9692 if (CFICount > 0 && CFICount != CFIInstructions.size())
9693 return std::nullopt;
9701 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
9702 !
MI.readsRegister(AArch64::SP, &
TRI))
9708 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
9713 if (
MI.mayLoadOrStore()) {
9716 bool OffsetIsScalable;
9720 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
9721 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
9725 if (OffsetIsScalable)
9733 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
9734 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
9737 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
9738 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
9753 bool AllStackInstrsSafe =
9758 if (RepeatedSequenceLocs[0].
back().isTerminator()) {
9760 NumBytesToCreateFrame = 0;
9761 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
9765 else if (LastInstrOpcode == AArch64::BL ||
9766 ((LastInstrOpcode == AArch64::BLR ||
9767 LastInstrOpcode == AArch64::BLRNoIP) &&
9771 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
9779 unsigned NumBytesNoStackCalls = 0;
9780 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
9786 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
9795 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
9798 if (LRAvailable && !IsNoReturn) {
9799 NumBytesNoStackCalls += 4;
9801 CandidatesWithoutStackFixups.push_back(
C);
9806 else if (findRegisterToSaveLRTo(
C)) {
9807 NumBytesNoStackCalls += 12;
9809 CandidatesWithoutStackFixups.push_back(
C);
9814 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
9815 NumBytesNoStackCalls += 12;
9817 CandidatesWithoutStackFixups.push_back(
C);
9823 NumBytesNoStackCalls += SequenceSize;
9830 if (!AllStackInstrsSafe ||
9831 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
9832 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
9834 if (RepeatedSequenceLocs.size() < MinRepeats)
9835 return std::nullopt;
9888 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
9889 !findRegisterToSaveLRTo(
C));
9895 if (RepeatedSequenceLocs.size() < MinRepeats)
9896 return std::nullopt;
9905 bool ModStackToSaveLR =
false;
9908 ModStackToSaveLR =
true;
9917 ModStackToSaveLR =
true;
9919 if (ModStackToSaveLR) {
9921 if (!AllStackInstrsSafe)
9922 return std::nullopt;
9925 NumBytesToCreateFrame += 8;
9932 return std::nullopt;
9934 return std::make_unique<outliner::OutlinedFunction>(
9935 RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID);
9938void AArch64InstrInfo::mergeOutliningCandidateAttributes(
9939 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
9943 const auto &CFn = Candidates.front().getMF()->getFunction();
9945 if (CFn.hasFnAttribute(
"ptrauth-returns"))
9946 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-returns"));
9947 if (CFn.hasFnAttribute(
"ptrauth-auth-traps"))
9948 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-auth-traps"));
9951 if (CFn.hasFnAttribute(
"sign-return-address"))
9952 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
9953 if (CFn.hasFnAttribute(
"sign-return-address-key"))
9954 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
9956 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
9959bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
9964 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
9977 AArch64FunctionInfo *AFI = MF.
getInfo<AArch64FunctionInfo>();
9978 if (!AFI || AFI->
hasRedZone().value_or(
true))
9998 unsigned &Flags)
const {
10000 "Must track liveness!");
10002 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10017 auto AreAllUnsafeRegsDead = [&LRU]() {
10018 return LRU.available(AArch64::W16) && LRU.available(AArch64::W17) &&
10019 LRU.available(AArch64::NZCV);
10034 bool LRAvailableEverywhere =
true;
10036 LRU.addLiveOuts(
MBB);
10038 auto UpdateWholeMBBFlags = [&
Flags](
const MachineInstr &
MI) {
10039 if (
MI.isCall() && !
MI.isTerminator())
10045 auto CreateNewRangeStartingAt =
10046 [&RangeBegin, &RangeEnd,
10048 RangeBegin = NewBegin;
10049 RangeEnd = std::next(RangeBegin);
10052 auto SaveRangeIfNonEmpty = [&RangeLen, &
Ranges, &RangeBegin, &RangeEnd]() {
10058 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
10060 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
10062 Ranges.emplace_back(RangeBegin, RangeEnd);
10070 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
10071 LRU.stepBackward(*FirstPossibleEndPt);
10074 UpdateWholeMBBFlags(*FirstPossibleEndPt);
10075 if (AreAllUnsafeRegsDead())
10082 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
10087 LRU.stepBackward(
MI);
10088 UpdateWholeMBBFlags(
MI);
10089 if (!AreAllUnsafeRegsDead()) {
10090 SaveRangeIfNonEmpty();
10091 CreateNewRangeStartingAt(
MI.getIterator());
10094 LRAvailableEverywhere &= LRU.available(AArch64::LR);
10095 RangeBegin =
MI.getIterator();
10100 if (AreAllUnsafeRegsDead())
10101 SaveRangeIfNonEmpty();
10109 if (!LRAvailableEverywhere)
10117 unsigned Flags)
const {
10118 MachineInstr &
MI = *MIT;
10122 switch (
MI.getOpcode()) {
10123 case AArch64::PACM:
10124 case AArch64::PACIASP:
10125 case AArch64::PACIBSP:
10126 case AArch64::PACIASPPC:
10127 case AArch64::PACIBSPPC:
10128 case AArch64::AUTIASP:
10129 case AArch64::AUTIBSP:
10130 case AArch64::AUTIASPPCi:
10131 case AArch64::AUTIASPPCr:
10132 case AArch64::AUTIBSPPCi:
10133 case AArch64::AUTIBSPPCr:
10134 case AArch64::RETAA:
10135 case AArch64::RETAB:
10136 case AArch64::RETAASPPCi:
10137 case AArch64::RETAASPPCr:
10138 case AArch64::RETABSPPCi:
10139 case AArch64::RETABSPPCr:
10140 case AArch64::EMITBKEY:
10141 case AArch64::PAUTH_PROLOGUE:
10142 case AArch64::PAUTH_EPILOGUE:
10152 if (
MI.isCFIInstruction())
10156 if (
MI.isTerminator())
10162 for (
const MachineOperand &MOP :
MI.operands()) {
10165 assert(!MOP.isCFIIndex());
10168 if (MOP.isReg() && !MOP.isImplicit() &&
10169 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
10176 if (
MI.getOpcode() == AArch64::ADRP)
10196 for (
const MachineOperand &MOP :
MI.operands()) {
10197 if (MOP.isGlobal()) {
10205 if (Callee &&
Callee->getName() ==
"\01_mcount")
10213 if (
MI.getOpcode() == AArch64::BLR ||
10214 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
10218 return UnknownCallOutlineType;
10226 return UnknownCallOutlineType;
10234 return UnknownCallOutlineType;
10255 for (MachineInstr &
MI :
MBB) {
10256 const MachineOperand *
Base;
10257 TypeSize Width(0,
false);
10259 bool OffsetIsScalable;
10262 if (!
MI.mayLoadOrStore() ||
10265 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
10269 TypeSize Scale(0U,
false);
10270 int64_t Dummy1, Dummy2;
10273 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
10275 assert(Scale != 0 &&
"Unexpected opcode!");
10276 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
10281 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
10282 StackOffsetOperand.
setImm(NewImm);
10288 bool ShouldSignReturnAddr) {
10289 if (!ShouldSignReturnAddr)
10295 TII->get(AArch64::PAUTH_EPILOGUE))
10299void AArch64InstrInfo::buildOutlinedFrame(
10303 AArch64FunctionInfo *FI = MF.
getInfo<AArch64FunctionInfo>();
10311 unsigned TailOpcode;
10313 TailOpcode = AArch64::TCRETURNdi;
10317 TailOpcode = AArch64::TCRETURNriALL;
10328 bool IsLeafFunction =
true;
10331 auto IsNonTailCall = [](
const MachineInstr &
MI) {
10332 return MI.isCall() && !
MI.isReturn();
10342 "Can only fix up stack references once");
10343 fixupPostOutline(
MBB);
10345 IsLeafFunction =
false;
10356 Et = std::prev(
MBB.
end());
10366 if (MF.
getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
10370 CFIBuilder.buildDefCFAOffset(16);
10374 CFIBuilder.buildOffset(AArch64::LR, -16);
10415 fixupPostOutline(
MBB);
10426 .addGlobalAddress(
M.getNamedValue(MF.
getName()))
10436 .addGlobalAddress(
M.getNamedValue(MF.
getName())));
10445 MachineInstr *Save;
10446 MachineInstr *Restore;
10452 assert(
Reg &&
"No callee-saved register available?");
10486 .addGlobalAddress(
M.getNamedValue(MF.
getName())));
10494bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
10502 bool AllowSideEffects)
const {
10504 const AArch64Subtarget &STI = MF.
getSubtarget<AArch64Subtarget>();
10507 if (
TRI.isGeneralPurposeRegister(MF,
Reg)) {
10520 assert(STI.hasNEON() &&
"Expected to have NEON.");
10526std::optional<DestSourcePair>
10531 if (((
MI.getOpcode() == AArch64::ORRWrs &&
10532 MI.getOperand(1).getReg() == AArch64::WZR &&
10533 MI.getOperand(3).getImm() == 0x0) ||
10534 (
MI.getOpcode() == AArch64::ORRWrr &&
10535 MI.getOperand(1).getReg() == AArch64::WZR)) &&
10537 (!
MI.getOperand(0).getReg().isVirtual() ||
10538 MI.getOperand(0).getSubReg() == 0) &&
10539 (!
MI.getOperand(0).getReg().isPhysical() ||
10544 if (
MI.getOpcode() == AArch64::ORRXrs &&
10545 MI.getOperand(1).getReg() == AArch64::XZR &&
10546 MI.getOperand(3).getImm() == 0x0)
10549 return std::nullopt;
10552std::optional<DestSourcePair>
10554 if ((
MI.getOpcode() == AArch64::ORRWrs &&
10555 MI.getOperand(1).getReg() == AArch64::WZR &&
10556 MI.getOperand(3).getImm() == 0x0) ||
10557 (
MI.getOpcode() == AArch64::ORRWrr &&
10558 MI.getOperand(1).getReg() == AArch64::WZR))
10560 return std::nullopt;
10563std::optional<RegImmPair>
10572 return std::nullopt;
10574 switch (
MI.getOpcode()) {
10576 return std::nullopt;
10577 case AArch64::SUBWri:
10578 case AArch64::SUBXri:
10579 case AArch64::SUBSWri:
10580 case AArch64::SUBSXri:
10583 case AArch64::ADDSWri:
10584 case AArch64::ADDSXri:
10585 case AArch64::ADDWri:
10586 case AArch64::ADDXri: {
10588 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
10589 !
MI.getOperand(2).isImm())
10590 return std::nullopt;
10591 int Shift =
MI.getOperand(3).getImm();
10592 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
10596 return RegImmPair{
MI.getOperand(1).getReg(),
Offset};
10602static std::optional<ParamLoadedValue>
10606 auto DestSrc =
TII->isCopyLikeInstr(
MI);
10608 return std::nullopt;
10610 Register DestReg = DestSrc->Destination->getReg();
10611 Register SrcReg = DestSrc->Source->getReg();
10616 if (DestReg == DescribedReg)
10620 if (
MI.getOpcode() == AArch64::ORRWrs &&
10621 TRI->isSuperRegister(DestReg, DescribedReg))
10625 if (
MI.getOpcode() == AArch64::ORRXrs &&
10626 TRI->isSubRegister(DestReg, DescribedReg)) {
10627 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
10631 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
10632 "Unhandled ORR[XW]rs copy case");
10634 return std::nullopt;
10637bool AArch64InstrInfo::isFunctionSafeToSplit(
const MachineFunction &MF)
const {
10642 if (MF.
getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(
true))
10648bool AArch64InstrInfo::isMBBSafeToSplitToCold(
10652 auto isAsmGoto = [](
const MachineInstr &
MI) {
10653 return MI.getOpcode() == AArch64::INLINEASM_BR;
10663 auto containsMBB = [&
MBB](
const MachineJumpTableEntry &JTE) {
10670 for (
const MachineInstr &
MI :
MBB) {
10671 switch (
MI.getOpcode()) {
10672 case TargetOpcode::G_BRJT:
10673 case AArch64::JumpTableDest32:
10674 case AArch64::JumpTableDest16:
10675 case AArch64::JumpTableDest8:
10686std::optional<ParamLoadedValue>
10689 const MachineFunction *MF =
MI.getMF();
10691 switch (
MI.getOpcode()) {
10692 case AArch64::MOVZWi:
10693 case AArch64::MOVZXi: {
10696 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(),
Reg))
10697 return std::nullopt;
10699 if (!
MI.getOperand(1).isImm())
10700 return std::nullopt;
10701 int64_t Immediate =
MI.getOperand(1).getImm();
10702 int Shift =
MI.getOperand(2).getImm();
10706 case AArch64::ORRWrs:
10707 case AArch64::ORRXrs:
10714bool AArch64InstrInfo::isExtendLikelyToBeFolded(
10717 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
10718 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
10721 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
10725 if (!
MRI.hasOneNonDBGUse(DefReg))
10730 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
10731 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
10734uint64_t AArch64InstrInfo::getElementSizeForOpcode(
unsigned Opc)
const {
10738bool AArch64InstrInfo::isPTestLikeOpcode(
unsigned Opc)
const {
10742bool AArch64InstrInfo::isWhileOpcode(
unsigned Opc)
const {
10747AArch64InstrInfo::getTailDuplicateSize(
CodeGenOptLevel OptLevel)
const {
10751bool AArch64InstrInfo::isLegalAddressingMode(
unsigned NumBytes, int64_t
Offset,
10752 unsigned Scale)
const {
10763 unsigned Shift =
Log2_64(NumBytes);
10764 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
10772 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
10777 return AArch64::BLRNoIP;
10779 return AArch64::BLR;
10784 Register TargetReg,
bool FrameSetup)
const {
10785 assert(TargetReg != AArch64::SP &&
"New top of stack cannot already be in SP");
10797 MF.
insert(MBBInsertPoint, LoopTestMBB);
10800 MF.
insert(MBBInsertPoint, LoopBodyMBB);
10802 MF.
insert(MBBInsertPoint, ExitMBB);
10812 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
10820 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::Bcc))
10826 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
10839 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
10858 MBB.addSuccessor(LoopTestMBB);
10864 return ExitMBB->
begin();
10881 unsigned CompCounterOprNum;
10885 unsigned UpdateCounterOprNum;
10889 bool IsUpdatePriorComp;
10901 TII(MF->getSubtarget().getInstrInfo()),
10902 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
10903 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
10904 CompCounterOprNum(CompCounterOprNum), Update(Update),
10905 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
10906 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
10908 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
10914 std::optional<bool> createTripCountGreaterCondition(
10915 int TC, MachineBasicBlock &
MBB,
10916 SmallVectorImpl<MachineOperand> &CondParam)
override {
10924 void createRemainingIterationsGreaterCondition(
10925 int TC, MachineBasicBlock &
MBB, SmallVectorImpl<MachineOperand> &
Cond,
10926 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts)
override;
10928 void setPreheader(MachineBasicBlock *NewPreheader)
override {}
10930 void adjustTripCount(
int TripCountAdjust)
override {}
10932 bool isMVEExpanderSupported()
override {
return true; }
10945 MBB.getParent()->getSubtarget().getRegisterInfo();
10950 Result =
MRI.createVirtualRegister(
10953 }
else if (
I == ReplaceOprNum) {
10954 MRI.constrainRegClass(ReplaceReg,
10959 MBB.insert(InsertTo, NewMI);
10963void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
10979 assert(CondBranch->getOpcode() == AArch64::Bcc);
10983 if (CondBranch->getOperand(1).getMBB() == LoopBB)
10990 auto AccumulateCond = [&](
Register CurCond,
10992 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
11001 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
11005 for (
int I = 0;
I <= TC; ++
I) {
11011 AccCond = AccumulateCond(AccCond, CC);
11015 if (Update != Comp && IsUpdatePriorComp) {
11017 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
11018 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
11022 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
11024 }
else if (Update != Comp) {
11029 Counter = NextCounter;
11033 if (LastStage0Insts.
empty()) {
11037 if (IsUpdatePriorComp)
11042 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
11045 for (
int I = 0;
I <= TC; ++
I) {
11049 AccCond = AccumulateCond(AccCond, CC);
11050 if (
I != TC && Update != Comp)
11053 Counter = NextCounter;
11069 assert(Phi.getNumOperands() == 5);
11070 if (Phi.getOperand(2).getMBB() ==
MBB) {
11071 RegMBB = Phi.getOperand(1).getReg();
11072 RegOther = Phi.getOperand(3).getReg();
11074 assert(Phi.getOperand(4).getMBB() ==
MBB);
11075 RegMBB = Phi.getOperand(3).getReg();
11076 RegOther = Phi.getOperand(1).getReg();
11081 if (!
Reg.isVirtual())
11084 return MRI.getVRegDef(
Reg)->getParent() != BB;
11090 unsigned &UpdateCounterOprNum,
Register &InitReg,
11091 bool &IsUpdatePriorComp) {
11105 if (!
Reg.isVirtual())
11108 UpdateInst =
nullptr;
11109 UpdateCounterOprNum = 0;
11111 IsUpdatePriorComp =
true;
11115 if (Def->getParent() != LoopBB)
11117 if (Def->isCopy()) {
11119 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
11121 CurReg = Def->getOperand(1).getReg();
11122 }
else if (Def->isPHI()) {
11126 IsUpdatePriorComp =
false;
11131 switch (Def->getOpcode()) {
11132 case AArch64::ADDSXri:
11133 case AArch64::ADDSWri:
11134 case AArch64::SUBSXri:
11135 case AArch64::SUBSWri:
11136 case AArch64::ADDXri:
11137 case AArch64::ADDWri:
11138 case AArch64::SUBXri:
11139 case AArch64::SUBWri:
11141 UpdateCounterOprNum = 1;
11143 case AArch64::ADDSXrr:
11144 case AArch64::ADDSWrr:
11145 case AArch64::SUBSXrr:
11146 case AArch64::SUBSWrr:
11147 case AArch64::ADDXrr:
11148 case AArch64::ADDWrr:
11149 case AArch64::SUBXrr:
11150 case AArch64::SUBWrr:
11153 UpdateCounterOprNum = 1;
11155 UpdateCounterOprNum = 2;
11162 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
11177std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
11188 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
11199 if (
TBB == LoopBB && FBB == LoopBB)
11203 if (
TBB != LoopBB && FBB ==
nullptr)
11206 assert((
TBB == LoopBB || FBB == LoopBB) &&
11207 "The Loop must be a single-basic-block loop");
11212 if (CondBranch->
getOpcode() != AArch64::Bcc)
11220 unsigned CompCounterOprNum = 0;
11222 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
11226 switch (
MI.getOpcode()) {
11227 case AArch64::SUBSXri:
11228 case AArch64::SUBSWri:
11229 case AArch64::ADDSXri:
11230 case AArch64::ADDSWri:
11232 CompCounterOprNum = 1;
11234 case AArch64::ADDSWrr:
11235 case AArch64::ADDSXrr:
11236 case AArch64::SUBSWrr:
11237 case AArch64::SUBSXrr:
11241 if (isWhileOpcode(
MI.getOpcode())) {
11248 if (CompCounterOprNum == 0) {
11250 CompCounterOprNum = 2;
11252 CompCounterOprNum = 1;
11264 bool IsUpdatePriorComp;
11265 unsigned UpdateCounterOprNum;
11267 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
11270 return std::make_unique<AArch64PipelinerLoopInfo>(
11271 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
11281 TypeSize Scale(0U,
false), Width(0U,
false);
11282 int64_t MinOffset, MaxOffset;
11283 if (
getMemOpInfo(
MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
11285 if (
MI.getOperand(ImmIdx).isImm() && !
MI.getOperand(ImmIdx - 1).isFI()) {
11286 int64_t Imm =
MI.getOperand(ImmIdx).getImm();
11287 if (Imm < MinOffset || Imm > MaxOffset) {
11288 ErrInfo =
"Unexpected immediate on load/store instruction";
11294 const MCInstrDesc &MCID =
MI.getDesc();
11296 const MachineOperand &MO =
MI.getOperand(
Op);
11300 ErrInfo =
"OPERAND_IMPLICIT_IMM_0 should be 0";
11309 ErrInfo =
"OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
11320#define GET_INSTRINFO_HELPERS
11321#define GET_INSTRMAP_INFO
11322#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static cl::opt< unsigned > GatherOptSearchLimit("aarch64-search-limit", cl::Hidden, cl::init(2048), cl::desc("Restrict range of instructions to search for the " "machine-combiner gather pattern optimization"))
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment, StringRef RegScale={})
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, MCRegister Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static bool isPostIndexLdStOpcode(unsigned Opcode)
Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
static unsigned getBranchDisplacementBits(unsigned Opc)
static cl::opt< unsigned > CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(9), cl::desc("Restrict range of CB instructions (DEBUG)"))
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static void appendReadRegExpr(SmallVectorImpl< char > &Expr, unsigned RegNum)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool getLoadPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Search for patterns of LD instructions we can optimize.
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static void appendLoadRegExpr(SmallVectorImpl< char > &Expr, int64_t OffsetFromDefCFA)
static void appendConstantExpr(SmallVectorImpl< char > &Expr, int64_t Constant, dwarf::LocationAtom Operation)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static void generateGatherLanePattern(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned Pattern, unsigned NumLanes)
Generate optimized instruction sequence for gather load patterns to improve Memory-Level Parallelism ...
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool getGatherLanePattern(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, unsigned LoadLaneOpCode, unsigned NumLanes)
Check if the given instruction forms a gather load pattern that can be optimized for better Memory-Le...
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
Machine Check Debug Module
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
MachineInstr unsigned OpIdx
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallSet class.
This file defines the SmallVector class.
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setOutliningStyle(const std::string &Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isAsCheapAsAMove(const MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This is an important base class in LLVM.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
Instructions::iterator instr_iterator
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
LLVM_ABI bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
LLVM_ABI uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
LLVM_ABI bool isLoadFoldBarrier() const
Returns true if it is illegal to fold a load across this instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
LLVM_ABI void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
LLVM_ABI void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
LLVM_ABI MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
LLVM_ABI MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Value * getOperand(unsigned i) const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_ARM64EC_CALLMANGLE
MO_ARM64EC_CALLMANGLE - Operand refers to the Arm64EC-mangled version of a symbol,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ ScalablePredicateVector
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
LLVM_ABI Instruction & back() const
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool succeeded(LogicalResult Result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
static bool isIndirectBranchOpcode(int Opc)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
static bool isSEHInstruction(const MachineInstr &MI)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
unsigned getUndefRegState(bool B)
static MCRegister getXRegFromWReg(MCRegister Reg)
unsigned getDefRegState(bool B)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA, std::optional< int64_t > IncomingVGOffsetFromDefCFA)
unsigned getKillRegState(bool B)
FunctionAddr VTableAddr Next
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
static bool isUncondBranchOpcode(int Opc)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
void appendLEB128(SmallVectorImpl< U > &Buffer, T Value)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
static const MachineMemOperand::Flags MOStridedAccess
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
LLVM_ABI static const MBBSectionID ColdSectionID
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.