65#define GET_INSTRINFO_CTOR_DTOR
66#include "AArch64GenInstrInfo.inc"
70 cl::desc(
"Restrict range of CB instructions (DEBUG)"));
74 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
78 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
82 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
86 cl::desc(
"Restrict range of B instructions (DEBUG)"));
90 cl::desc(
"Restrict range of instructions to search for the "
91 "machine-combiner gather pattern optimization"));
96 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
107 auto Op =
MI.getOpcode();
108 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
109 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
113 if (
MI.isMetaInstruction())
118 unsigned NumBytes = 0;
122 NumBytes =
Desc.getSize() ?
Desc.getSize() : 4;
125 if (!MFI->shouldSignReturnAddress(MF))
129 auto Method = STI.getAuthenticatedLRCheckMethod(*MF);
137 switch (
Desc.getOpcode()) {
140 return Desc.getSize();
147 case TargetOpcode::STACKMAP:
150 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
152 case TargetOpcode::PATCHPOINT:
155 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
157 case TargetOpcode::STATEPOINT:
159 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
164 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
169 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
171 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
172 case TargetOpcode::PATCHABLE_TAIL_CALL:
173 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
177 case TargetOpcode::PATCHABLE_EVENT_CALL:
183 NumBytes =
MI.getOperand(1).getImm();
185 case TargetOpcode::BUNDLE:
186 NumBytes = getInstBundleLength(
MI);
193unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
197 while (++
I != E &&
I->isInsideBundle()) {
198 assert(!
I->isBundle() &&
"No nested bundle!");
233 case AArch64::CBWPri:
234 case AArch64::CBXPri:
235 case AArch64::CBWPrr:
236 case AArch64::CBXPrr:
265 case AArch64::CBWPri:
266 case AArch64::CBXPri:
267 case AArch64::CBWPrr:
268 case AArch64::CBXPrr:
274 int64_t BrOffset)
const {
276 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
277 "over conditional branch expansion");
278 return isIntN(Bits, BrOffset / 4);
283 switch (
MI.getOpcode()) {
287 return MI.getOperand(0).getMBB();
292 return MI.getOperand(2).getMBB();
298 return MI.getOperand(1).getMBB();
299 case AArch64::CBWPri:
300 case AArch64::CBXPri:
301 case AArch64::CBWPrr:
302 case AArch64::CBXPrr:
303 return MI.getOperand(3).getMBB();
313 assert(RS &&
"RegScavenger required for long branching");
315 "new block should be inserted for expanding unconditional branch");
318 "restore block should be inserted for restoring clobbered registers");
325 "Branch offsets outside of the signed 33-bit range not supported");
336 RS->enterBasicBlockEnd(
MBB);
339 constexpr Register Reg = AArch64::X16;
340 if (!RS->isRegUsed(Reg)) {
341 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
348 Register Scavenged = RS->FindUnusedReg(&AArch64::GPR64RegClass);
349 if (Scavenged != AArch64::NoRegister &&
351 buildIndirectBranch(Scavenged, NewDestBB);
352 RS->setRegUsed(Scavenged);
361 "Unable to insert indirect branch inside function that has red zone");
384 bool AllowModify)
const {
391 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
392 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
396 if (!isUnpredicatedTerminator(*
I))
403 unsigned LastOpc = LastInst->
getOpcode();
404 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
419 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
426 LastInst = SecondLastInst;
428 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
433 SecondLastInst = &*
I;
434 SecondLastOpc = SecondLastInst->
getOpcode();
445 LastInst = SecondLastInst;
447 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
449 "unreachable unconditional branches removed above");
458 SecondLastInst = &*
I;
459 SecondLastOpc = SecondLastInst->
getOpcode();
463 if (SecondLastInst &&
I !=
MBB.begin() && isUnpredicatedTerminator(*--
I))
479 I->eraseFromParent();
488 I->eraseFromParent();
497 MachineBranchPredicate &MBP,
498 bool AllowModify)
const {
508 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
509 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
513 if (!isUnpredicatedTerminator(*
I))
518 unsigned LastOpc = LastInst->
getOpcode();
533 assert(MBP.TrueDest &&
"expected!");
534 MBP.FalseDest =
MBB.getNextNode();
536 MBP.ConditionDef =
nullptr;
537 MBP.SingleUseCondition =
false;
541 MBP.Predicate = (LastOpc == AArch64::CBNZX || LastOpc == AArch64::CBNZW)
542 ? MachineBranchPredicate::PRED_NE
543 : MachineBranchPredicate::PRED_EQ;
559 Cond[1].setImm(AArch64::CBNZW);
562 Cond[1].setImm(AArch64::CBZW);
565 Cond[1].setImm(AArch64::CBNZX);
568 Cond[1].setImm(AArch64::CBZX);
571 Cond[1].setImm(AArch64::TBNZW);
574 Cond[1].setImm(AArch64::TBZW);
577 Cond[1].setImm(AArch64::TBNZX);
580 Cond[1].setImm(AArch64::TBZX);
584 case AArch64::CBWPri:
585 case AArch64::CBXPri:
586 case AArch64::CBWPrr:
587 case AArch64::CBXPrr: {
600 int *BytesRemoved)
const {
610 I->eraseFromParent();
614 if (
I ==
MBB.begin()) {
627 I->eraseFromParent();
634void AArch64InstrInfo::instantiateCondBranch(
664 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
692 if (!
DefMI->isFullCopy())
694 VReg =
DefMI->getOperand(1).getReg();
703 unsigned *NewReg =
nullptr) {
708 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
712 switch (
DefMI->getOpcode()) {
713 case AArch64::SUBREG_TO_REG:
717 if (!
DefMI->getOperand(1).isImm() ||
DefMI->getOperand(1).getImm() != 0)
719 if (!
DefMI->getOperand(2).isReg())
721 if (!
DefMI->getOperand(3).isImm() ||
722 DefMI->getOperand(3).getImm() != AArch64::sub_32)
725 if (
DefMI->getOpcode() != AArch64::MOVi32imm)
727 if (!
DefMI->getOperand(1).isImm() ||
DefMI->getOperand(1).getImm() != 1)
730 SrcReg = AArch64::XZR;
731 Opc = AArch64::CSINCXr;
734 case AArch64::MOVi32imm:
735 case AArch64::MOVi64imm:
736 if (!
DefMI->getOperand(1).isImm() ||
DefMI->getOperand(1).getImm() != 1)
738 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
739 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
742 case AArch64::ADDSXri:
743 case AArch64::ADDSWri:
745 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
750 case AArch64::ADDXri:
751 case AArch64::ADDWri:
753 if (!
DefMI->getOperand(2).isImm() ||
DefMI->getOperand(2).getImm() != 1 ||
754 DefMI->getOperand(3).getImm() != 0)
756 SrcReg =
DefMI->getOperand(1).getReg();
757 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
760 case AArch64::ORNXrr:
761 case AArch64::ORNWrr: {
764 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
766 SrcReg =
DefMI->getOperand(2).getReg();
767 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
771 case AArch64::SUBSXrr:
772 case AArch64::SUBSWrr:
774 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
779 case AArch64::SUBXrr:
780 case AArch64::SUBWrr: {
783 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
785 SrcReg =
DefMI->getOperand(2).getReg();
786 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
792 assert(
Opc && SrcReg &&
"Missing parameters");
804 int &FalseCycles)
const {
808 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
815 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
819 unsigned ExtraCondLat =
Cond.size() != 1;
823 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
824 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
826 CondCycles = 1 + ExtraCondLat;
827 TrueCycles = FalseCycles = 1;
837 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
838 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
839 CondCycles = 5 + ExtraCondLat;
840 TrueCycles = FalseCycles = 2;
857 switch (
Cond.size()) {
889 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
895 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
934 unsigned SUBSOpC, SUBSDestReg;
940 case AArch64::CBWPri:
941 SUBSOpC = AArch64::SUBSWri;
942 SUBSDestReg = AArch64::WZR;
945 case AArch64::CBXPri:
946 SUBSOpC = AArch64::SUBSXri;
947 SUBSDestReg = AArch64::XZR;
950 case AArch64::CBWPrr:
951 SUBSOpC = AArch64::SUBSWrr;
952 SUBSDestReg = AArch64::WZR;
955 case AArch64::CBXPrr:
956 SUBSOpC = AArch64::SUBSXrr;
957 SUBSDestReg = AArch64::XZR;
976 bool TryFold =
false;
977 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
978 RC = &AArch64::GPR64RegClass;
979 Opc = AArch64::CSELXr;
981 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
982 RC = &AArch64::GPR32RegClass;
983 Opc = AArch64::CSELWr;
985 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
986 RC = &AArch64::FPR64RegClass;
987 Opc = AArch64::FCSELDrrr;
988 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
989 RC = &AArch64::FPR32RegClass;
990 Opc = AArch64::FCSELSrrr;
992 assert(RC &&
"Unsupported regclass");
1011 MRI.clearKillFlags(NewReg);
1016 MRI.constrainRegClass(TrueReg, RC);
1019 (FalseReg.
isVirtual() || FalseReg == AArch64::WZR ||
1020 FalseReg == AArch64::XZR) &&
1021 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1023 MRI.constrainRegClass(FalseReg, RC);
1038 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
1043 return Is.
size() <= 2;
1049 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1050 if (isExynosCheapAsMove(
MI))
1052 return MI.isAsCheapAsAMove();
1055 switch (
MI.getOpcode()) {
1057 return MI.isAsCheapAsAMove();
1059 case AArch64::ADDWrs:
1060 case AArch64::ADDXrs:
1061 case AArch64::SUBWrs:
1062 case AArch64::SUBXrs:
1063 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
1068 case AArch64::MOVi32imm:
1070 case AArch64::MOVi64imm:
1075bool AArch64InstrInfo::isFalkorShiftExtFast(
const MachineInstr &
MI) {
1076 switch (
MI.getOpcode()) {
1080 case AArch64::ADDWrs:
1081 case AArch64::ADDXrs:
1082 case AArch64::ADDSWrs:
1083 case AArch64::ADDSXrs: {
1084 unsigned Imm =
MI.getOperand(3).getImm();
1091 case AArch64::ADDWrx:
1092 case AArch64::ADDXrx:
1093 case AArch64::ADDXrx64:
1094 case AArch64::ADDSWrx:
1095 case AArch64::ADDSXrx:
1096 case AArch64::ADDSXrx64: {
1097 unsigned Imm =
MI.getOperand(3).getImm();
1109 case AArch64::SUBWrs:
1110 case AArch64::SUBSWrs: {
1111 unsigned Imm =
MI.getOperand(3).getImm();
1113 return ShiftVal == 0 ||
1117 case AArch64::SUBXrs:
1118 case AArch64::SUBSXrs: {
1119 unsigned Imm =
MI.getOperand(3).getImm();
1121 return ShiftVal == 0 ||
1125 case AArch64::SUBWrx:
1126 case AArch64::SUBXrx:
1127 case AArch64::SUBXrx64:
1128 case AArch64::SUBSWrx:
1129 case AArch64::SUBSXrx:
1130 case AArch64::SUBSXrx64: {
1131 unsigned Imm =
MI.getOperand(3).getImm();
1143 case AArch64::LDRBBroW:
1144 case AArch64::LDRBBroX:
1145 case AArch64::LDRBroW:
1146 case AArch64::LDRBroX:
1147 case AArch64::LDRDroW:
1148 case AArch64::LDRDroX:
1149 case AArch64::LDRHHroW:
1150 case AArch64::LDRHHroX:
1151 case AArch64::LDRHroW:
1152 case AArch64::LDRHroX:
1153 case AArch64::LDRQroW:
1154 case AArch64::LDRQroX:
1155 case AArch64::LDRSBWroW:
1156 case AArch64::LDRSBWroX:
1157 case AArch64::LDRSBXroW:
1158 case AArch64::LDRSBXroX:
1159 case AArch64::LDRSHWroW:
1160 case AArch64::LDRSHWroX:
1161 case AArch64::LDRSHXroW:
1162 case AArch64::LDRSHXroX:
1163 case AArch64::LDRSWroW:
1164 case AArch64::LDRSWroX:
1165 case AArch64::LDRSroW:
1166 case AArch64::LDRSroX:
1167 case AArch64::LDRWroW:
1168 case AArch64::LDRWroX:
1169 case AArch64::LDRXroW:
1170 case AArch64::LDRXroX:
1171 case AArch64::PRFMroW:
1172 case AArch64::PRFMroX:
1173 case AArch64::STRBBroW:
1174 case AArch64::STRBBroX:
1175 case AArch64::STRBroW:
1176 case AArch64::STRBroX:
1177 case AArch64::STRDroW:
1178 case AArch64::STRDroX:
1179 case AArch64::STRHHroW:
1180 case AArch64::STRHHroX:
1181 case AArch64::STRHroW:
1182 case AArch64::STRHroX:
1183 case AArch64::STRQroW:
1184 case AArch64::STRQroX:
1185 case AArch64::STRSroW:
1186 case AArch64::STRSroX:
1187 case AArch64::STRWroW:
1188 case AArch64::STRWroX:
1189 case AArch64::STRXroW:
1190 case AArch64::STRXroX: {
1191 unsigned IsSigned =
MI.getOperand(3).getImm();
1198 unsigned Opc =
MI.getOpcode();
1202 case AArch64::SEH_StackAlloc:
1203 case AArch64::SEH_SaveFPLR:
1204 case AArch64::SEH_SaveFPLR_X:
1205 case AArch64::SEH_SaveReg:
1206 case AArch64::SEH_SaveReg_X:
1207 case AArch64::SEH_SaveRegP:
1208 case AArch64::SEH_SaveRegP_X:
1209 case AArch64::SEH_SaveFReg:
1210 case AArch64::SEH_SaveFReg_X:
1211 case AArch64::SEH_SaveFRegP:
1212 case AArch64::SEH_SaveFRegP_X:
1213 case AArch64::SEH_SetFP:
1214 case AArch64::SEH_AddFP:
1215 case AArch64::SEH_Nop:
1216 case AArch64::SEH_PrologEnd:
1217 case AArch64::SEH_EpilogStart:
1218 case AArch64::SEH_EpilogEnd:
1219 case AArch64::SEH_PACSignLR:
1220 case AArch64::SEH_SaveAnyRegQP:
1221 case AArch64::SEH_SaveAnyRegQPX:
1222 case AArch64::SEH_AllocZ:
1223 case AArch64::SEH_SaveZReg:
1224 case AArch64::SEH_SavePReg:
1231 unsigned &SubIdx)
const {
1232 switch (
MI.getOpcode()) {
1235 case AArch64::SBFMXri:
1236 case AArch64::UBFMXri:
1239 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1242 SrcReg =
MI.getOperand(1).getReg();
1243 DstReg =
MI.getOperand(0).getReg();
1244 SubIdx = AArch64::sub_32;
1253 int64_t OffsetA = 0, OffsetB = 0;
1254 TypeSize WidthA(0,
false), WidthB(0,
false);
1255 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1276 OffsetAIsScalable == OffsetBIsScalable) {
1277 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1278 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1279 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1280 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1298 switch (
MI.getOpcode()) {
1301 if (
MI.getOperand(0).getImm() == 0x14)
1308 case AArch64::MSRpstatesvcrImm1:
1315 auto Next = std::next(
MI.getIterator());
1316 return Next !=
MBB->end() &&
Next->isCFIInstruction();
1323 Register &SrcReg2, int64_t &CmpMask,
1324 int64_t &CmpValue)
const {
1328 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1329 if (!
MI.getOperand(1).isReg() ||
MI.getOperand(1).getSubReg())
1332 switch (
MI.getOpcode()) {
1335 case AArch64::PTEST_PP:
1336 case AArch64::PTEST_PP_ANY:
1337 case AArch64::PTEST_PP_FIRST:
1338 SrcReg =
MI.getOperand(0).getReg();
1339 SrcReg2 =
MI.getOperand(1).getReg();
1340 if (
MI.getOperand(2).getSubReg())
1347 case AArch64::SUBSWrr:
1348 case AArch64::SUBSWrs:
1349 case AArch64::SUBSWrx:
1350 case AArch64::SUBSXrr:
1351 case AArch64::SUBSXrs:
1352 case AArch64::SUBSXrx:
1353 case AArch64::ADDSWrr:
1354 case AArch64::ADDSWrs:
1355 case AArch64::ADDSWrx:
1356 case AArch64::ADDSXrr:
1357 case AArch64::ADDSXrs:
1358 case AArch64::ADDSXrx:
1360 SrcReg =
MI.getOperand(1).getReg();
1361 SrcReg2 =
MI.getOperand(2).getReg();
1364 if (
MI.getOperand(2).getSubReg())
1370 case AArch64::SUBSWri:
1371 case AArch64::ADDSWri:
1372 case AArch64::SUBSXri:
1373 case AArch64::ADDSXri:
1374 SrcReg =
MI.getOperand(1).getReg();
1377 CmpValue =
MI.getOperand(2).getImm();
1379 case AArch64::ANDSWri:
1380 case AArch64::ANDSXri:
1383 SrcReg =
MI.getOperand(1).getReg();
1387 MI.getOperand(2).getImm(),
1388 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1397 assert(
MBB &&
"Can't get MachineBasicBlock here");
1399 assert(MF &&
"Can't get MachineFunction here");
1404 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands();
OpIdx < EndIdx;
1411 if (!OpRegCstraints)
1419 "Operand has register constraints without being a register!");
1422 if (
Reg.isPhysical()) {
1426 !
MRI->constrainRegClass(
Reg, OpRegCstraints))
1439 bool MIDefinesZeroReg =
false;
1440 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1441 MI.definesRegister(AArch64::XZR,
nullptr))
1442 MIDefinesZeroReg =
true;
1444 switch (
MI.getOpcode()) {
1446 return MI.getOpcode();
1447 case AArch64::ADDSWrr:
1448 return AArch64::ADDWrr;
1449 case AArch64::ADDSWri:
1450 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1451 case AArch64::ADDSWrs:
1452 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1453 case AArch64::ADDSWrx:
1454 return AArch64::ADDWrx;
1455 case AArch64::ADDSXrr:
1456 return AArch64::ADDXrr;
1457 case AArch64::ADDSXri:
1458 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1459 case AArch64::ADDSXrs:
1460 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1461 case AArch64::ADDSXrx:
1462 return AArch64::ADDXrx;
1463 case AArch64::SUBSWrr:
1464 return AArch64::SUBWrr;
1465 case AArch64::SUBSWri:
1466 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1467 case AArch64::SUBSWrs:
1468 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1469 case AArch64::SUBSWrx:
1470 return AArch64::SUBWrx;
1471 case AArch64::SUBSXrr:
1472 return AArch64::SUBXrr;
1473 case AArch64::SUBSXri:
1474 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1475 case AArch64::SUBSXrs:
1476 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1477 case AArch64::SUBSXrx:
1478 return AArch64::SUBXrx;
1493 if (To == To->getParent()->begin())
1498 if (To->getParent() != From->getParent())
1510 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1511 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1517std::optional<unsigned>
1521 unsigned MaskOpcode =
Mask->getOpcode();
1522 unsigned PredOpcode = Pred->
getOpcode();
1523 bool PredIsPTestLike = isPTestLikeOpcode(PredOpcode);
1524 bool PredIsWhileLike = isWhileOpcode(PredOpcode);
1526 if (PredIsWhileLike) {
1530 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1537 getElementSizeForOpcode(MaskOpcode) ==
1538 getElementSizeForOpcode(PredOpcode))
1544 if (PTest->
getOpcode() == AArch64::PTEST_PP_FIRST &&
1551 if (PredIsPTestLike) {
1556 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1564 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1565 PTestLikeMask->getOperand(1).getReg().isVirtual())
1567 MRI->getUniqueVRegDef(PTestLikeMask->getOperand(1).getReg());
1573 getElementSizeForOpcode(MaskOpcode) ==
1574 getElementSizeForOpcode(PredOpcode)) {
1575 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1601 uint64_t PredElementSize = getElementSizeForOpcode(PredOpcode);
1603 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1611 switch (PredOpcode) {
1612 case AArch64::AND_PPzPP:
1613 case AArch64::BIC_PPzPP:
1614 case AArch64::EOR_PPzPP:
1615 case AArch64::NAND_PPzPP:
1616 case AArch64::NOR_PPzPP:
1617 case AArch64::ORN_PPzPP:
1618 case AArch64::ORR_PPzPP:
1619 case AArch64::BRKA_PPzP:
1620 case AArch64::BRKPA_PPzPP:
1621 case AArch64::BRKB_PPzP:
1622 case AArch64::BRKPB_PPzPP:
1623 case AArch64::RDFFR_PPz: {
1627 if (Mask != PredMask)
1631 case AArch64::BRKN_PPzP: {
1635 if ((MaskOpcode != AArch64::PTRUE_B) ||
1636 (
Mask->getOperand(1).getImm() != 31))
1640 case AArch64::PTRUE_B:
1653bool AArch64InstrInfo::optimizePTestInstr(
1654 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1656 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1657 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1659 if (Pred->
isCopy() && PTest->
getOpcode() == AArch64::PTEST_PP_FIRST) {
1663 if (
Op.isReg() &&
Op.getReg().isVirtual() &&
1664 Op.getSubReg() == AArch64::psub0)
1665 Pred =
MRI->getUniqueVRegDef(
Op.getReg());
1668 unsigned PredOpcode = Pred->
getOpcode();
1669 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1685 if (*NewOp != PredOpcode) {
1696 for (; i !=
e; ++i) {
1727 if (DeadNZCVIdx != -1) {
1746 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1747 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY ||
1748 CmpInstr.
getOpcode() == AArch64::PTEST_PP_FIRST)
1749 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1758 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1760 return (CmpValue == 0 || CmpValue == 1) &&
1761 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1769 switch (Instr.getOpcode()) {
1771 return AArch64::INSTRUCTION_LIST_END;
1773 case AArch64::ADDSWrr:
1774 case AArch64::ADDSWri:
1775 case AArch64::ADDSXrr:
1776 case AArch64::ADDSXri:
1777 case AArch64::SUBSWrr:
1778 case AArch64::SUBSWri:
1779 case AArch64::SUBSXrr:
1780 case AArch64::SUBSXri:
1781 return Instr.getOpcode();
1783 case AArch64::ADDWrr:
1784 return AArch64::ADDSWrr;
1785 case AArch64::ADDWri:
1786 return AArch64::ADDSWri;
1787 case AArch64::ADDXrr:
1788 return AArch64::ADDSXrr;
1789 case AArch64::ADDXri:
1790 return AArch64::ADDSXri;
1791 case AArch64::ADCWr:
1792 return AArch64::ADCSWr;
1793 case AArch64::ADCXr:
1794 return AArch64::ADCSXr;
1795 case AArch64::SUBWrr:
1796 return AArch64::SUBSWrr;
1797 case AArch64::SUBWri:
1798 return AArch64::SUBSWri;
1799 case AArch64::SUBXrr:
1800 return AArch64::SUBSXrr;
1801 case AArch64::SUBXri:
1802 return AArch64::SUBSXri;
1803 case AArch64::SBCWr:
1804 return AArch64::SBCSWr;
1805 case AArch64::SBCXr:
1806 return AArch64::SBCSXr;
1807 case AArch64::ANDWri:
1808 return AArch64::ANDSWri;
1809 case AArch64::ANDXri:
1810 return AArch64::ANDSXri;
1816 for (
auto *BB :
MBB->successors())
1817 if (BB->isLiveIn(AArch64::NZCV))
1826 switch (Instr.getOpcode()) {
1830 case AArch64::Bcc: {
1831 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1836 case AArch64::CSINVWr:
1837 case AArch64::CSINVXr:
1838 case AArch64::CSINCWr:
1839 case AArch64::CSINCXr:
1840 case AArch64::CSELWr:
1841 case AArch64::CSELXr:
1842 case AArch64::CSNEGWr:
1843 case AArch64::CSNEGXr:
1844 case AArch64::FCSELSrrr:
1845 case AArch64::FCSELDrrr: {
1846 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1859 Instr.getOperand(CCIdx).
getImm())
1912std::optional<UsedNZCV>
1917 if (
MI.getParent() != CmpParent)
1918 return std::nullopt;
1921 return std::nullopt;
1926 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1929 return std::nullopt;
1934 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1937 return NZCVUsedAfterCmp;
1941 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1945 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1967 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1973 "Caller guarantees that CmpInstr compares with constant 0");
1976 if (!NZVCUsed || NZVCUsed->C)
1998bool AArch64InstrInfo::substituteCmpToZero(
2002 MachineInstr *
MI =
MRI.getUniqueVRegDef(SrcReg);
2009 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2016 MI->setDesc(
get(NewOpc));
2021 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
2033 assert((CmpValue == 0 || CmpValue == 1) &&
2034 "Only comparisons to 0 or 1 considered for removal!");
2037 unsigned MIOpc =
MI.getOpcode();
2038 if (MIOpc == AArch64::CSINCWr) {
2039 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
2040 MI.getOperand(2).getReg() != AArch64::WZR)
2042 }
else if (MIOpc == AArch64::CSINCXr) {
2043 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
2044 MI.getOperand(2).getReg() != AArch64::XZR)
2054 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
2058 const unsigned CmpOpcode = CmpInstr.
getOpcode();
2060 if (CmpValue && !IsSubsRegImm)
2062 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
2067 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
2070 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2074 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2077 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
2078 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
2081 if (MIUsedNZCV.
N && !CmpValue)
2123bool AArch64InstrInfo::removeCmpToZeroOrOne(
2126 MachineInstr *
MI =
MRI.getUniqueVRegDef(SrcReg);
2130 SmallVector<MachineInstr *, 4> CCUseInstrs;
2131 bool IsInvertCC =
false;
2139 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2141 assert(Idx >= 0 &&
"Unexpected instruction using CC.");
2142 MachineOperand &CCOperand = CCUseInstr->getOperand(Idx);
2151bool AArch64InstrInfo::expandPostRAPseudo(
MachineInstr &
MI)
const {
2152 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2153 MI.getOpcode() != AArch64::CATCHRET)
2156 MachineBasicBlock &
MBB = *
MI.getParent();
2158 auto TRI = Subtarget.getRegisterInfo();
2161 if (
MI.getOpcode() == AArch64::CATCHRET) {
2163 const TargetInstrInfo *
TII =
2165 MachineBasicBlock *TargetMBB =
MI.getOperand(0).getMBB();
2170 FirstEpilogSEH = std::prev(FirstEpilogSEH);
2172 FirstEpilogSEH = std::next(FirstEpilogSEH);
2187 if (
M.getStackProtectorGuard() ==
"sysreg") {
2188 const AArch64SysReg::SysReg *SrcReg =
2189 AArch64SysReg::lookupSysRegByName(
M.getStackProtectorGuardReg());
2197 int Offset =
M.getStackProtectorGuardOffset();
2248 const GlobalValue *GV =
2251 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2257 if (Subtarget.isTargetILP32()) {
2258 unsigned Reg32 =
TRI->getSubReg(
Reg, AArch64::sub_32);
2272 assert(!Subtarget.isTargetILP32() &&
"how can large exist in ILP32?");
2299 if (Subtarget.isTargetILP32()) {
2300 unsigned Reg32 =
TRI->getSubReg(
Reg, AArch64::sub_32);
2323 switch (
MI.getOpcode()) {
2326 case AArch64::MOVZWi:
2327 case AArch64::MOVZXi:
2328 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2329 assert(
MI.getDesc().getNumOperands() == 3 &&
2330 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2334 case AArch64::ANDWri:
2335 return MI.getOperand(1).getReg() == AArch64::WZR;
2336 case AArch64::ANDXri:
2337 return MI.getOperand(1).getReg() == AArch64::XZR;
2338 case TargetOpcode::COPY:
2339 return MI.getOperand(1).getReg() == AArch64::WZR;
2347 switch (
MI.getOpcode()) {
2350 case TargetOpcode::COPY: {
2353 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2354 AArch64::GPR64RegClass.
contains(DstReg));
2356 case AArch64::ORRXrs:
2357 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2358 assert(
MI.getDesc().getNumOperands() == 4 &&
2359 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2363 case AArch64::ADDXri:
2364 if (
MI.getOperand(2).getImm() == 0) {
2365 assert(
MI.getDesc().getNumOperands() == 4 &&
2366 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2377 switch (
MI.getOpcode()) {
2380 case TargetOpcode::COPY: {
2382 return AArch64::FPR128RegClass.contains(DstReg);
2384 case AArch64::ORRv16i8:
2385 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2386 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2387 "invalid ORRv16i8 operands");
2396 int &FrameIndex)
const {
2397 switch (
MI.getOpcode()) {
2400 case AArch64::LDRWui:
2401 case AArch64::LDRXui:
2402 case AArch64::LDRBui:
2403 case AArch64::LDRHui:
2404 case AArch64::LDRSui:
2405 case AArch64::LDRDui:
2406 case AArch64::LDRQui:
2407 case AArch64::LDR_PXI:
2408 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2409 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2410 FrameIndex =
MI.getOperand(1).getIndex();
2411 return MI.getOperand(0).getReg();
2420 int &FrameIndex)
const {
2421 switch (
MI.getOpcode()) {
2424 case AArch64::STRWui:
2425 case AArch64::STRXui:
2426 case AArch64::STRBui:
2427 case AArch64::STRHui:
2428 case AArch64::STRSui:
2429 case AArch64::STRDui:
2430 case AArch64::STRQui:
2431 case AArch64::STR_PXI:
2432 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2433 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2434 FrameIndex =
MI.getOperand(1).getIndex();
2435 return MI.getOperand(0).getReg();
2445 return MMO->getFlags() & MOSuppressPair;
2451 if (
MI.memoperands_empty())
2459 return MMO->getFlags() & MOStridedAccess;
2467 case AArch64::STURSi:
2468 case AArch64::STRSpre:
2469 case AArch64::STURDi:
2470 case AArch64::STRDpre:
2471 case AArch64::STURQi:
2472 case AArch64::STRQpre:
2473 case AArch64::STURBBi:
2474 case AArch64::STURHHi:
2475 case AArch64::STURWi:
2476 case AArch64::STRWpre:
2477 case AArch64::STURXi:
2478 case AArch64::STRXpre:
2479 case AArch64::LDURSi:
2480 case AArch64::LDRSpre:
2481 case AArch64::LDURDi:
2482 case AArch64::LDRDpre:
2483 case AArch64::LDURQi:
2484 case AArch64::LDRQpre:
2485 case AArch64::LDURWi:
2486 case AArch64::LDRWpre:
2487 case AArch64::LDURXi:
2488 case AArch64::LDRXpre:
2489 case AArch64::LDRSWpre:
2490 case AArch64::LDURSWi:
2491 case AArch64::LDURHHi:
2492 case AArch64::LDURBBi:
2493 case AArch64::LDURSBWi:
2494 case AArch64::LDURSHWi:
2502 case AArch64::PRFMui:
return AArch64::PRFUMi;
2503 case AArch64::LDRXui:
return AArch64::LDURXi;
2504 case AArch64::LDRWui:
return AArch64::LDURWi;
2505 case AArch64::LDRBui:
return AArch64::LDURBi;
2506 case AArch64::LDRHui:
return AArch64::LDURHi;
2507 case AArch64::LDRSui:
return AArch64::LDURSi;
2508 case AArch64::LDRDui:
return AArch64::LDURDi;
2509 case AArch64::LDRQui:
return AArch64::LDURQi;
2510 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2511 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2512 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2513 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2514 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2515 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2516 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2517 case AArch64::STRXui:
return AArch64::STURXi;
2518 case AArch64::STRWui:
return AArch64::STURWi;
2519 case AArch64::STRBui:
return AArch64::STURBi;
2520 case AArch64::STRHui:
return AArch64::STURHi;
2521 case AArch64::STRSui:
return AArch64::STURSi;
2522 case AArch64::STRDui:
return AArch64::STURDi;
2523 case AArch64::STRQui:
return AArch64::STURQi;
2524 case AArch64::STRBBui:
return AArch64::STURBBi;
2525 case AArch64::STRHHui:
return AArch64::STURHHi;
2534 case AArch64::LDAPURBi:
2535 case AArch64::LDAPURHi:
2536 case AArch64::LDAPURi:
2537 case AArch64::LDAPURSBWi:
2538 case AArch64::LDAPURSBXi:
2539 case AArch64::LDAPURSHWi:
2540 case AArch64::LDAPURSHXi:
2541 case AArch64::LDAPURSWi:
2542 case AArch64::LDAPURXi:
2543 case AArch64::LDR_PPXI:
2544 case AArch64::LDR_PXI:
2545 case AArch64::LDR_ZXI:
2546 case AArch64::LDR_ZZXI:
2547 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
2548 case AArch64::LDR_ZZZXI:
2549 case AArch64::LDR_ZZZZXI:
2550 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
2551 case AArch64::LDRBBui:
2552 case AArch64::LDRBui:
2553 case AArch64::LDRDui:
2554 case AArch64::LDRHHui:
2555 case AArch64::LDRHui:
2556 case AArch64::LDRQui:
2557 case AArch64::LDRSBWui:
2558 case AArch64::LDRSBXui:
2559 case AArch64::LDRSHWui:
2560 case AArch64::LDRSHXui:
2561 case AArch64::LDRSui:
2562 case AArch64::LDRSWui:
2563 case AArch64::LDRWui:
2564 case AArch64::LDRXui:
2565 case AArch64::LDURBBi:
2566 case AArch64::LDURBi:
2567 case AArch64::LDURDi:
2568 case AArch64::LDURHHi:
2569 case AArch64::LDURHi:
2570 case AArch64::LDURQi:
2571 case AArch64::LDURSBWi:
2572 case AArch64::LDURSBXi:
2573 case AArch64::LDURSHWi:
2574 case AArch64::LDURSHXi:
2575 case AArch64::LDURSi:
2576 case AArch64::LDURSWi:
2577 case AArch64::LDURWi:
2578 case AArch64::LDURXi:
2579 case AArch64::PRFMui:
2580 case AArch64::PRFUMi:
2581 case AArch64::ST2Gi:
2583 case AArch64::STLURBi:
2584 case AArch64::STLURHi:
2585 case AArch64::STLURWi:
2586 case AArch64::STLURXi:
2587 case AArch64::StoreSwiftAsyncContext:
2588 case AArch64::STR_PPXI:
2589 case AArch64::STR_PXI:
2590 case AArch64::STR_ZXI:
2591 case AArch64::STR_ZZXI:
2592 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
2593 case AArch64::STR_ZZZXI:
2594 case AArch64::STR_ZZZZXI:
2595 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
2596 case AArch64::STRBBui:
2597 case AArch64::STRBui:
2598 case AArch64::STRDui:
2599 case AArch64::STRHHui:
2600 case AArch64::STRHui:
2601 case AArch64::STRQui:
2602 case AArch64::STRSui:
2603 case AArch64::STRWui:
2604 case AArch64::STRXui:
2605 case AArch64::STURBBi:
2606 case AArch64::STURBi:
2607 case AArch64::STURDi:
2608 case AArch64::STURHHi:
2609 case AArch64::STURHi:
2610 case AArch64::STURQi:
2611 case AArch64::STURSi:
2612 case AArch64::STURWi:
2613 case AArch64::STURXi:
2614 case AArch64::STZ2Gi:
2615 case AArch64::STZGi:
2616 case AArch64::TAGPstack:
2618 case AArch64::LD1B_D_IMM:
2619 case AArch64::LD1B_H_IMM:
2620 case AArch64::LD1B_IMM:
2621 case AArch64::LD1B_S_IMM:
2622 case AArch64::LD1D_IMM:
2623 case AArch64::LD1H_D_IMM:
2624 case AArch64::LD1H_IMM:
2625 case AArch64::LD1H_S_IMM:
2626 case AArch64::LD1RB_D_IMM:
2627 case AArch64::LD1RB_H_IMM:
2628 case AArch64::LD1RB_IMM:
2629 case AArch64::LD1RB_S_IMM:
2630 case AArch64::LD1RD_IMM:
2631 case AArch64::LD1RH_D_IMM:
2632 case AArch64::LD1RH_IMM:
2633 case AArch64::LD1RH_S_IMM:
2634 case AArch64::LD1RSB_D_IMM:
2635 case AArch64::LD1RSB_H_IMM:
2636 case AArch64::LD1RSB_S_IMM:
2637 case AArch64::LD1RSH_D_IMM:
2638 case AArch64::LD1RSH_S_IMM:
2639 case AArch64::LD1RSW_IMM:
2640 case AArch64::LD1RW_D_IMM:
2641 case AArch64::LD1RW_IMM:
2642 case AArch64::LD1SB_D_IMM:
2643 case AArch64::LD1SB_H_IMM:
2644 case AArch64::LD1SB_S_IMM:
2645 case AArch64::LD1SH_D_IMM:
2646 case AArch64::LD1SH_S_IMM:
2647 case AArch64::LD1SW_D_IMM:
2648 case AArch64::LD1W_D_IMM:
2649 case AArch64::LD1W_IMM:
2650 case AArch64::LD2B_IMM:
2651 case AArch64::LD2D_IMM:
2652 case AArch64::LD2H_IMM:
2653 case AArch64::LD2W_IMM:
2654 case AArch64::LD3B_IMM:
2655 case AArch64::LD3D_IMM:
2656 case AArch64::LD3H_IMM:
2657 case AArch64::LD3W_IMM:
2658 case AArch64::LD4B_IMM:
2659 case AArch64::LD4D_IMM:
2660 case AArch64::LD4H_IMM:
2661 case AArch64::LD4W_IMM:
2663 case AArch64::LDNF1B_D_IMM:
2664 case AArch64::LDNF1B_H_IMM:
2665 case AArch64::LDNF1B_IMM:
2666 case AArch64::LDNF1B_S_IMM:
2667 case AArch64::LDNF1D_IMM:
2668 case AArch64::LDNF1H_D_IMM:
2669 case AArch64::LDNF1H_IMM:
2670 case AArch64::LDNF1H_S_IMM:
2671 case AArch64::LDNF1SB_D_IMM:
2672 case AArch64::LDNF1SB_H_IMM:
2673 case AArch64::LDNF1SB_S_IMM:
2674 case AArch64::LDNF1SH_D_IMM:
2675 case AArch64::LDNF1SH_S_IMM:
2676 case AArch64::LDNF1SW_D_IMM:
2677 case AArch64::LDNF1W_D_IMM:
2678 case AArch64::LDNF1W_IMM:
2679 case AArch64::LDNPDi:
2680 case AArch64::LDNPQi:
2681 case AArch64::LDNPSi:
2682 case AArch64::LDNPWi:
2683 case AArch64::LDNPXi:
2684 case AArch64::LDNT1B_ZRI:
2685 case AArch64::LDNT1D_ZRI:
2686 case AArch64::LDNT1H_ZRI:
2687 case AArch64::LDNT1W_ZRI:
2688 case AArch64::LDPDi:
2689 case AArch64::LDPQi:
2690 case AArch64::LDPSi:
2691 case AArch64::LDPWi:
2692 case AArch64::LDPXi:
2693 case AArch64::LDRBBpost:
2694 case AArch64::LDRBBpre:
2695 case AArch64::LDRBpost:
2696 case AArch64::LDRBpre:
2697 case AArch64::LDRDpost:
2698 case AArch64::LDRDpre:
2699 case AArch64::LDRHHpost:
2700 case AArch64::LDRHHpre:
2701 case AArch64::LDRHpost:
2702 case AArch64::LDRHpre:
2703 case AArch64::LDRQpost:
2704 case AArch64::LDRQpre:
2705 case AArch64::LDRSpost:
2706 case AArch64::LDRSpre:
2707 case AArch64::LDRWpost:
2708 case AArch64::LDRWpre:
2709 case AArch64::LDRXpost:
2710 case AArch64::LDRXpre:
2711 case AArch64::ST1B_D_IMM:
2712 case AArch64::ST1B_H_IMM:
2713 case AArch64::ST1B_IMM:
2714 case AArch64::ST1B_S_IMM:
2715 case AArch64::ST1D_IMM:
2716 case AArch64::ST1H_D_IMM:
2717 case AArch64::ST1H_IMM:
2718 case AArch64::ST1H_S_IMM:
2719 case AArch64::ST1W_D_IMM:
2720 case AArch64::ST1W_IMM:
2721 case AArch64::ST2B_IMM:
2722 case AArch64::ST2D_IMM:
2723 case AArch64::ST2H_IMM:
2724 case AArch64::ST2W_IMM:
2725 case AArch64::ST3B_IMM:
2726 case AArch64::ST3D_IMM:
2727 case AArch64::ST3H_IMM:
2728 case AArch64::ST3W_IMM:
2729 case AArch64::ST4B_IMM:
2730 case AArch64::ST4D_IMM:
2731 case AArch64::ST4H_IMM:
2732 case AArch64::ST4W_IMM:
2733 case AArch64::STGPi:
2734 case AArch64::STGPreIndex:
2735 case AArch64::STZGPreIndex:
2736 case AArch64::ST2GPreIndex:
2737 case AArch64::STZ2GPreIndex:
2738 case AArch64::STGPostIndex:
2739 case AArch64::STZGPostIndex:
2740 case AArch64::ST2GPostIndex:
2741 case AArch64::STZ2GPostIndex:
2742 case AArch64::STNPDi:
2743 case AArch64::STNPQi:
2744 case AArch64::STNPSi:
2745 case AArch64::STNPWi:
2746 case AArch64::STNPXi:
2747 case AArch64::STNT1B_ZRI:
2748 case AArch64::STNT1D_ZRI:
2749 case AArch64::STNT1H_ZRI:
2750 case AArch64::STNT1W_ZRI:
2751 case AArch64::STPDi:
2752 case AArch64::STPQi:
2753 case AArch64::STPSi:
2754 case AArch64::STPWi:
2755 case AArch64::STPXi:
2756 case AArch64::STRBBpost:
2757 case AArch64::STRBBpre:
2758 case AArch64::STRBpost:
2759 case AArch64::STRBpre:
2760 case AArch64::STRDpost:
2761 case AArch64::STRDpre:
2762 case AArch64::STRHHpost:
2763 case AArch64::STRHHpre:
2764 case AArch64::STRHpost:
2765 case AArch64::STRHpre:
2766 case AArch64::STRQpost:
2767 case AArch64::STRQpre:
2768 case AArch64::STRSpost:
2769 case AArch64::STRSpre:
2770 case AArch64::STRWpost:
2771 case AArch64::STRWpre:
2772 case AArch64::STRXpost:
2773 case AArch64::STRXpre:
2775 case AArch64::LDPDpost:
2776 case AArch64::LDPDpre:
2777 case AArch64::LDPQpost:
2778 case AArch64::LDPQpre:
2779 case AArch64::LDPSpost:
2780 case AArch64::LDPSpre:
2781 case AArch64::LDPWpost:
2782 case AArch64::LDPWpre:
2783 case AArch64::LDPXpost:
2784 case AArch64::LDPXpre:
2785 case AArch64::STGPpre:
2786 case AArch64::STGPpost:
2787 case AArch64::STPDpost:
2788 case AArch64::STPDpre:
2789 case AArch64::STPQpost:
2790 case AArch64::STPQpre:
2791 case AArch64::STPSpost:
2792 case AArch64::STPSpre:
2793 case AArch64::STPWpost:
2794 case AArch64::STPWpre:
2795 case AArch64::STPXpost:
2796 case AArch64::STPXpre:
2802 switch (
MI.getOpcode()) {
2806 case AArch64::STRSui:
2807 case AArch64::STRDui:
2808 case AArch64::STRQui:
2809 case AArch64::STRXui:
2810 case AArch64::STRWui:
2811 case AArch64::LDRSui:
2812 case AArch64::LDRDui:
2813 case AArch64::LDRQui:
2814 case AArch64::LDRXui:
2815 case AArch64::LDRWui:
2816 case AArch64::LDRSWui:
2818 case AArch64::STURSi:
2819 case AArch64::STRSpre:
2820 case AArch64::STURDi:
2821 case AArch64::STRDpre:
2822 case AArch64::STURQi:
2823 case AArch64::STRQpre:
2824 case AArch64::STURWi:
2825 case AArch64::STRWpre:
2826 case AArch64::STURXi:
2827 case AArch64::STRXpre:
2828 case AArch64::LDURSi:
2829 case AArch64::LDRSpre:
2830 case AArch64::LDURDi:
2831 case AArch64::LDRDpre:
2832 case AArch64::LDURQi:
2833 case AArch64::LDRQpre:
2834 case AArch64::LDURWi:
2835 case AArch64::LDRWpre:
2836 case AArch64::LDURXi:
2837 case AArch64::LDRXpre:
2838 case AArch64::LDURSWi:
2839 case AArch64::LDRSWpre:
2841 case AArch64::LDR_ZXI:
2842 case AArch64::STR_ZXI:
2848 switch (
MI.getOpcode()) {
2851 "Unexpected instruction - was a new tail call opcode introduced?");
2853 case AArch64::TCRETURNdi:
2854 case AArch64::TCRETURNri:
2855 case AArch64::TCRETURNrix16x17:
2856 case AArch64::TCRETURNrix17:
2857 case AArch64::TCRETURNrinotx16:
2858 case AArch64::TCRETURNriALL:
2859 case AArch64::AUTH_TCRETURN:
2860 case AArch64::AUTH_TCRETURN_BTI:
2870 case AArch64::ADDWri:
2871 return AArch64::ADDSWri;
2872 case AArch64::ADDWrr:
2873 return AArch64::ADDSWrr;
2874 case AArch64::ADDWrs:
2875 return AArch64::ADDSWrs;
2876 case AArch64::ADDWrx:
2877 return AArch64::ADDSWrx;
2878 case AArch64::ANDWri:
2879 return AArch64::ANDSWri;
2880 case AArch64::ANDWrr:
2881 return AArch64::ANDSWrr;
2882 case AArch64::ANDWrs:
2883 return AArch64::ANDSWrs;
2884 case AArch64::BICWrr:
2885 return AArch64::BICSWrr;
2886 case AArch64::BICWrs:
2887 return AArch64::BICSWrs;
2888 case AArch64::SUBWri:
2889 return AArch64::SUBSWri;
2890 case AArch64::SUBWrr:
2891 return AArch64::SUBSWrr;
2892 case AArch64::SUBWrs:
2893 return AArch64::SUBSWrs;
2894 case AArch64::SUBWrx:
2895 return AArch64::SUBSWrx;
2897 case AArch64::ADDXri:
2898 return AArch64::ADDSXri;
2899 case AArch64::ADDXrr:
2900 return AArch64::ADDSXrr;
2901 case AArch64::ADDXrs:
2902 return AArch64::ADDSXrs;
2903 case AArch64::ADDXrx:
2904 return AArch64::ADDSXrx;
2905 case AArch64::ANDXri:
2906 return AArch64::ANDSXri;
2907 case AArch64::ANDXrr:
2908 return AArch64::ANDSXrr;
2909 case AArch64::ANDXrs:
2910 return AArch64::ANDSXrs;
2911 case AArch64::BICXrr:
2912 return AArch64::BICSXrr;
2913 case AArch64::BICXrs:
2914 return AArch64::BICSXrs;
2915 case AArch64::SUBXri:
2916 return AArch64::SUBSXri;
2917 case AArch64::SUBXrr:
2918 return AArch64::SUBSXrr;
2919 case AArch64::SUBXrs:
2920 return AArch64::SUBSXrs;
2921 case AArch64::SUBXrx:
2922 return AArch64::SUBSXrx;
2924 case AArch64::AND_PPzPP:
2925 return AArch64::ANDS_PPzPP;
2926 case AArch64::BIC_PPzPP:
2927 return AArch64::BICS_PPzPP;
2928 case AArch64::EOR_PPzPP:
2929 return AArch64::EORS_PPzPP;
2930 case AArch64::NAND_PPzPP:
2931 return AArch64::NANDS_PPzPP;
2932 case AArch64::NOR_PPzPP:
2933 return AArch64::NORS_PPzPP;
2934 case AArch64::ORN_PPzPP:
2935 return AArch64::ORNS_PPzPP;
2936 case AArch64::ORR_PPzPP:
2937 return AArch64::ORRS_PPzPP;
2938 case AArch64::BRKA_PPzP:
2939 return AArch64::BRKAS_PPzP;
2940 case AArch64::BRKPA_PPzPP:
2941 return AArch64::BRKPAS_PPzPP;
2942 case AArch64::BRKB_PPzP:
2943 return AArch64::BRKBS_PPzP;
2944 case AArch64::BRKPB_PPzPP:
2945 return AArch64::BRKPBS_PPzPP;
2946 case AArch64::BRKN_PPzP:
2947 return AArch64::BRKNS_PPzP;
2948 case AArch64::RDFFR_PPz:
2949 return AArch64::RDFFRS_PPz;
2950 case AArch64::PTRUE_B:
2951 return AArch64::PTRUES_B;
2962 if (
MI.hasOrderedMemoryRef())
2967 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2968 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2969 "Expected a reg or frame index operand.");
2973 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2975 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2988 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2989 Register BaseReg =
MI.getOperand(1).getReg();
2991 if (
MI.modifiesRegister(BaseReg,
TRI))
2997 switch (
MI.getOpcode()) {
3000 case AArch64::LDR_ZXI:
3001 case AArch64::STR_ZXI:
3002 if (!Subtarget.isLittleEndian() ||
3003 Subtarget.getSVEVectorSizeInBits() != 128)
3016 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
3018 MI.getMF()->getFunction().needsUnwindTableEntry();
3024 if (Subtarget.isPaired128Slow()) {
3025 switch (
MI.getOpcode()) {
3028 case AArch64::LDURQi:
3029 case AArch64::STURQi:
3030 case AArch64::LDRQui:
3031 case AArch64::STRQui:
3058std::optional<ExtAddrMode>
3063 bool OffsetIsScalable;
3064 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
3065 return std::nullopt;
3068 return std::nullopt;
3083 int64_t OffsetScale = 1;
3088 case AArch64::LDURQi:
3089 case AArch64::STURQi:
3093 case AArch64::LDURDi:
3094 case AArch64::STURDi:
3095 case AArch64::LDURXi:
3096 case AArch64::STURXi:
3100 case AArch64::LDURWi:
3101 case AArch64::LDURSWi:
3102 case AArch64::STURWi:
3106 case AArch64::LDURHi:
3107 case AArch64::STURHi:
3108 case AArch64::LDURHHi:
3109 case AArch64::STURHHi:
3110 case AArch64::LDURSHXi:
3111 case AArch64::LDURSHWi:
3115 case AArch64::LDRBroX:
3116 case AArch64::LDRBBroX:
3117 case AArch64::LDRSBXroX:
3118 case AArch64::LDRSBWroX:
3119 case AArch64::STRBroX:
3120 case AArch64::STRBBroX:
3121 case AArch64::LDURBi:
3122 case AArch64::LDURBBi:
3123 case AArch64::LDURSBXi:
3124 case AArch64::LDURSBWi:
3125 case AArch64::STURBi:
3126 case AArch64::STURBBi:
3127 case AArch64::LDRBui:
3128 case AArch64::LDRBBui:
3129 case AArch64::LDRSBXui:
3130 case AArch64::LDRSBWui:
3131 case AArch64::STRBui:
3132 case AArch64::STRBBui:
3136 case AArch64::LDRQroX:
3137 case AArch64::STRQroX:
3138 case AArch64::LDRQui:
3139 case AArch64::STRQui:
3144 case AArch64::LDRDroX:
3145 case AArch64::STRDroX:
3146 case AArch64::LDRXroX:
3147 case AArch64::STRXroX:
3148 case AArch64::LDRDui:
3149 case AArch64::STRDui:
3150 case AArch64::LDRXui:
3151 case AArch64::STRXui:
3156 case AArch64::LDRWroX:
3157 case AArch64::LDRSWroX:
3158 case AArch64::STRWroX:
3159 case AArch64::LDRWui:
3160 case AArch64::LDRSWui:
3161 case AArch64::STRWui:
3166 case AArch64::LDRHroX:
3167 case AArch64::STRHroX:
3168 case AArch64::LDRHHroX:
3169 case AArch64::STRHHroX:
3170 case AArch64::LDRSHXroX:
3171 case AArch64::LDRSHWroX:
3172 case AArch64::LDRHui:
3173 case AArch64::STRHui:
3174 case AArch64::LDRHHui:
3175 case AArch64::STRHHui:
3176 case AArch64::LDRSHXui:
3177 case AArch64::LDRSHWui:
3185 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
3209 case AArch64::SBFMXri:
3222 AM.
Scale = OffsetScale;
3227 case TargetOpcode::SUBREG_TO_REG: {
3240 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3244 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3245 DefMI.getOperand(1).getReg() != AArch64::WZR ||
3246 DefMI.getOperand(3).getImm() != 0)
3253 AM.
Scale = OffsetScale;
3264 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3265 int64_t NewOffset) ->
bool {
3266 int64_t MinOffset, MaxOffset;
3283 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3284 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3286 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3288 int64_t NewOffset = OldOffset + Disp;
3289 if (!isLegalAddressingMode(NumBytes, NewOffset, 0))
3293 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3303 auto canFoldAddRegIntoAddrMode =
3308 if ((
unsigned)Scale != Scale)
3310 if (!isLegalAddressingMode(NumBytes, 0, Scale))
3322 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3323 Subtarget.isSTRQroSlow();
3332 case AArch64::ADDXri:
3338 return canFoldAddSubImmIntoAddrMode(Disp);
3340 case AArch64::SUBXri:
3346 return canFoldAddSubImmIntoAddrMode(-Disp);
3348 case AArch64::ADDXrs: {
3361 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3363 if (avoidSlowSTRQ(MemI))
3366 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3369 case AArch64::ADDXrr:
3377 if (!OptSize && avoidSlowSTRQ(MemI))
3379 return canFoldAddRegIntoAddrMode(1);
3381 case AArch64::ADDXrx:
3389 if (!OptSize && avoidSlowSTRQ(MemI))
3398 return canFoldAddRegIntoAddrMode(
3413 case AArch64::LDURQi:
3414 case AArch64::LDRQui:
3415 return AArch64::LDRQroX;
3416 case AArch64::STURQi:
3417 case AArch64::STRQui:
3418 return AArch64::STRQroX;
3419 case AArch64::LDURDi:
3420 case AArch64::LDRDui:
3421 return AArch64::LDRDroX;
3422 case AArch64::STURDi:
3423 case AArch64::STRDui:
3424 return AArch64::STRDroX;
3425 case AArch64::LDURXi:
3426 case AArch64::LDRXui:
3427 return AArch64::LDRXroX;
3428 case AArch64::STURXi:
3429 case AArch64::STRXui:
3430 return AArch64::STRXroX;
3431 case AArch64::LDURWi:
3432 case AArch64::LDRWui:
3433 return AArch64::LDRWroX;
3434 case AArch64::LDURSWi:
3435 case AArch64::LDRSWui:
3436 return AArch64::LDRSWroX;
3437 case AArch64::STURWi:
3438 case AArch64::STRWui:
3439 return AArch64::STRWroX;
3440 case AArch64::LDURHi:
3441 case AArch64::LDRHui:
3442 return AArch64::LDRHroX;
3443 case AArch64::STURHi:
3444 case AArch64::STRHui:
3445 return AArch64::STRHroX;
3446 case AArch64::LDURHHi:
3447 case AArch64::LDRHHui:
3448 return AArch64::LDRHHroX;
3449 case AArch64::STURHHi:
3450 case AArch64::STRHHui:
3451 return AArch64::STRHHroX;
3452 case AArch64::LDURSHXi:
3453 case AArch64::LDRSHXui:
3454 return AArch64::LDRSHXroX;
3455 case AArch64::LDURSHWi:
3456 case AArch64::LDRSHWui:
3457 return AArch64::LDRSHWroX;
3458 case AArch64::LDURBi:
3459 case AArch64::LDRBui:
3460 return AArch64::LDRBroX;
3461 case AArch64::LDURBBi:
3462 case AArch64::LDRBBui:
3463 return AArch64::LDRBBroX;
3464 case AArch64::LDURSBXi:
3465 case AArch64::LDRSBXui:
3466 return AArch64::LDRSBXroX;
3467 case AArch64::LDURSBWi:
3468 case AArch64::LDRSBWui:
3469 return AArch64::LDRSBWroX;
3470 case AArch64::STURBi:
3471 case AArch64::STRBui:
3472 return AArch64::STRBroX;
3473 case AArch64::STURBBi:
3474 case AArch64::STRBBui:
3475 return AArch64::STRBBroX;
3487 case AArch64::LDURQi:
3489 return AArch64::LDRQui;
3490 case AArch64::STURQi:
3492 return AArch64::STRQui;
3493 case AArch64::LDURDi:
3495 return AArch64::LDRDui;
3496 case AArch64::STURDi:
3498 return AArch64::STRDui;
3499 case AArch64::LDURXi:
3501 return AArch64::LDRXui;
3502 case AArch64::STURXi:
3504 return AArch64::STRXui;
3505 case AArch64::LDURWi:
3507 return AArch64::LDRWui;
3508 case AArch64::LDURSWi:
3510 return AArch64::LDRSWui;
3511 case AArch64::STURWi:
3513 return AArch64::STRWui;
3514 case AArch64::LDURHi:
3516 return AArch64::LDRHui;
3517 case AArch64::STURHi:
3519 return AArch64::STRHui;
3520 case AArch64::LDURHHi:
3522 return AArch64::LDRHHui;
3523 case AArch64::STURHHi:
3525 return AArch64::STRHHui;
3526 case AArch64::LDURSHXi:
3528 return AArch64::LDRSHXui;
3529 case AArch64::LDURSHWi:
3531 return AArch64::LDRSHWui;
3532 case AArch64::LDURBi:
3534 return AArch64::LDRBui;
3535 case AArch64::LDURBBi:
3537 return AArch64::LDRBBui;
3538 case AArch64::LDURSBXi:
3540 return AArch64::LDRSBXui;
3541 case AArch64::LDURSBWi:
3543 return AArch64::LDRSBWui;
3544 case AArch64::STURBi:
3546 return AArch64::STRBui;
3547 case AArch64::STURBBi:
3549 return AArch64::STRBBui;
3550 case AArch64::LDRQui:
3551 case AArch64::STRQui:
3554 case AArch64::LDRDui:
3555 case AArch64::STRDui:
3556 case AArch64::LDRXui:
3557 case AArch64::STRXui:
3560 case AArch64::LDRWui:
3561 case AArch64::LDRSWui:
3562 case AArch64::STRWui:
3565 case AArch64::LDRHui:
3566 case AArch64::STRHui:
3567 case AArch64::LDRHHui:
3568 case AArch64::STRHHui:
3569 case AArch64::LDRSHXui:
3570 case AArch64::LDRSHWui:
3573 case AArch64::LDRBui:
3574 case AArch64::LDRBBui:
3575 case AArch64::LDRSBXui:
3576 case AArch64::LDRSBWui:
3577 case AArch64::STRBui:
3578 case AArch64::STRBBui:
3592 case AArch64::LDURQi:
3593 case AArch64::STURQi:
3594 case AArch64::LDURDi:
3595 case AArch64::STURDi:
3596 case AArch64::LDURXi:
3597 case AArch64::STURXi:
3598 case AArch64::LDURWi:
3599 case AArch64::LDURSWi:
3600 case AArch64::STURWi:
3601 case AArch64::LDURHi:
3602 case AArch64::STURHi:
3603 case AArch64::LDURHHi:
3604 case AArch64::STURHHi:
3605 case AArch64::LDURSHXi:
3606 case AArch64::LDURSHWi:
3607 case AArch64::LDURBi:
3608 case AArch64::STURBi:
3609 case AArch64::LDURBBi:
3610 case AArch64::STURBBi:
3611 case AArch64::LDURSBWi:
3612 case AArch64::LDURSBXi:
3614 case AArch64::LDRQui:
3615 return AArch64::LDURQi;
3616 case AArch64::STRQui:
3617 return AArch64::STURQi;
3618 case AArch64::LDRDui:
3619 return AArch64::LDURDi;
3620 case AArch64::STRDui:
3621 return AArch64::STURDi;
3622 case AArch64::LDRXui:
3623 return AArch64::LDURXi;
3624 case AArch64::STRXui:
3625 return AArch64::STURXi;
3626 case AArch64::LDRWui:
3627 return AArch64::LDURWi;
3628 case AArch64::LDRSWui:
3629 return AArch64::LDURSWi;
3630 case AArch64::STRWui:
3631 return AArch64::STURWi;
3632 case AArch64::LDRHui:
3633 return AArch64::LDURHi;
3634 case AArch64::STRHui:
3635 return AArch64::STURHi;
3636 case AArch64::LDRHHui:
3637 return AArch64::LDURHHi;
3638 case AArch64::STRHHui:
3639 return AArch64::STURHHi;
3640 case AArch64::LDRSHXui:
3641 return AArch64::LDURSHXi;
3642 case AArch64::LDRSHWui:
3643 return AArch64::LDURSHWi;
3644 case AArch64::LDRBBui:
3645 return AArch64::LDURBBi;
3646 case AArch64::LDRBui:
3647 return AArch64::LDURBi;
3648 case AArch64::STRBBui:
3649 return AArch64::STURBBi;
3650 case AArch64::STRBui:
3651 return AArch64::STURBi;
3652 case AArch64::LDRSBWui:
3653 return AArch64::LDURSBWi;
3654 case AArch64::LDRSBXui:
3655 return AArch64::LDURSBXi;
3668 case AArch64::LDRQroX:
3669 case AArch64::LDURQi:
3670 case AArch64::LDRQui:
3671 return AArch64::LDRQroW;
3672 case AArch64::STRQroX:
3673 case AArch64::STURQi:
3674 case AArch64::STRQui:
3675 return AArch64::STRQroW;
3676 case AArch64::LDRDroX:
3677 case AArch64::LDURDi:
3678 case AArch64::LDRDui:
3679 return AArch64::LDRDroW;
3680 case AArch64::STRDroX:
3681 case AArch64::STURDi:
3682 case AArch64::STRDui:
3683 return AArch64::STRDroW;
3684 case AArch64::LDRXroX:
3685 case AArch64::LDURXi:
3686 case AArch64::LDRXui:
3687 return AArch64::LDRXroW;
3688 case AArch64::STRXroX:
3689 case AArch64::STURXi:
3690 case AArch64::STRXui:
3691 return AArch64::STRXroW;
3692 case AArch64::LDRWroX:
3693 case AArch64::LDURWi:
3694 case AArch64::LDRWui:
3695 return AArch64::LDRWroW;
3696 case AArch64::LDRSWroX:
3697 case AArch64::LDURSWi:
3698 case AArch64::LDRSWui:
3699 return AArch64::LDRSWroW;
3700 case AArch64::STRWroX:
3701 case AArch64::STURWi:
3702 case AArch64::STRWui:
3703 return AArch64::STRWroW;
3704 case AArch64::LDRHroX:
3705 case AArch64::LDURHi:
3706 case AArch64::LDRHui:
3707 return AArch64::LDRHroW;
3708 case AArch64::STRHroX:
3709 case AArch64::STURHi:
3710 case AArch64::STRHui:
3711 return AArch64::STRHroW;
3712 case AArch64::LDRHHroX:
3713 case AArch64::LDURHHi:
3714 case AArch64::LDRHHui:
3715 return AArch64::LDRHHroW;
3716 case AArch64::STRHHroX:
3717 case AArch64::STURHHi:
3718 case AArch64::STRHHui:
3719 return AArch64::STRHHroW;
3720 case AArch64::LDRSHXroX:
3721 case AArch64::LDURSHXi:
3722 case AArch64::LDRSHXui:
3723 return AArch64::LDRSHXroW;
3724 case AArch64::LDRSHWroX:
3725 case AArch64::LDURSHWi:
3726 case AArch64::LDRSHWui:
3727 return AArch64::LDRSHWroW;
3728 case AArch64::LDRBroX:
3729 case AArch64::LDURBi:
3730 case AArch64::LDRBui:
3731 return AArch64::LDRBroW;
3732 case AArch64::LDRBBroX:
3733 case AArch64::LDURBBi:
3734 case AArch64::LDRBBui:
3735 return AArch64::LDRBBroW;
3736 case AArch64::LDRSBXroX:
3737 case AArch64::LDURSBXi:
3738 case AArch64::LDRSBXui:
3739 return AArch64::LDRSBXroW;
3740 case AArch64::LDRSBWroX:
3741 case AArch64::LDURSBWi:
3742 case AArch64::LDRSBWui:
3743 return AArch64::LDRSBWroW;
3744 case AArch64::STRBroX:
3745 case AArch64::STURBi:
3746 case AArch64::STRBui:
3747 return AArch64::STRBroW;
3748 case AArch64::STRBBroX:
3749 case AArch64::STURBBi:
3750 case AArch64::STRBBui:
3751 return AArch64::STRBBroW;
3766 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3776 return B.getInstr();
3780 "Addressing mode not supported for folding");
3797 return B.getInstr();
3804 "Address offset can be a register or an immediate, but not both");
3806 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3811 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3825 return B.getInstr();
3829 "Function must not be called with an addressing mode it can't handle");
3838 case AArch64::LD1Fourv16b_POST:
3839 case AArch64::LD1Fourv1d_POST:
3840 case AArch64::LD1Fourv2d_POST:
3841 case AArch64::LD1Fourv2s_POST:
3842 case AArch64::LD1Fourv4h_POST:
3843 case AArch64::LD1Fourv4s_POST:
3844 case AArch64::LD1Fourv8b_POST:
3845 case AArch64::LD1Fourv8h_POST:
3846 case AArch64::LD1Onev16b_POST:
3847 case AArch64::LD1Onev1d_POST:
3848 case AArch64::LD1Onev2d_POST:
3849 case AArch64::LD1Onev2s_POST:
3850 case AArch64::LD1Onev4h_POST:
3851 case AArch64::LD1Onev4s_POST:
3852 case AArch64::LD1Onev8b_POST:
3853 case AArch64::LD1Onev8h_POST:
3854 case AArch64::LD1Rv16b_POST:
3855 case AArch64::LD1Rv1d_POST:
3856 case AArch64::LD1Rv2d_POST:
3857 case AArch64::LD1Rv2s_POST:
3858 case AArch64::LD1Rv4h_POST:
3859 case AArch64::LD1Rv4s_POST:
3860 case AArch64::LD1Rv8b_POST:
3861 case AArch64::LD1Rv8h_POST:
3862 case AArch64::LD1Threev16b_POST:
3863 case AArch64::LD1Threev1d_POST:
3864 case AArch64::LD1Threev2d_POST:
3865 case AArch64::LD1Threev2s_POST:
3866 case AArch64::LD1Threev4h_POST:
3867 case AArch64::LD1Threev4s_POST:
3868 case AArch64::LD1Threev8b_POST:
3869 case AArch64::LD1Threev8h_POST:
3870 case AArch64::LD1Twov16b_POST:
3871 case AArch64::LD1Twov1d_POST:
3872 case AArch64::LD1Twov2d_POST:
3873 case AArch64::LD1Twov2s_POST:
3874 case AArch64::LD1Twov4h_POST:
3875 case AArch64::LD1Twov4s_POST:
3876 case AArch64::LD1Twov8b_POST:
3877 case AArch64::LD1Twov8h_POST:
3878 case AArch64::LD1i16_POST:
3879 case AArch64::LD1i32_POST:
3880 case AArch64::LD1i64_POST:
3881 case AArch64::LD1i8_POST:
3882 case AArch64::LD2Rv16b_POST:
3883 case AArch64::LD2Rv1d_POST:
3884 case AArch64::LD2Rv2d_POST:
3885 case AArch64::LD2Rv2s_POST:
3886 case AArch64::LD2Rv4h_POST:
3887 case AArch64::LD2Rv4s_POST:
3888 case AArch64::LD2Rv8b_POST:
3889 case AArch64::LD2Rv8h_POST:
3890 case AArch64::LD2Twov16b_POST:
3891 case AArch64::LD2Twov2d_POST:
3892 case AArch64::LD2Twov2s_POST:
3893 case AArch64::LD2Twov4h_POST:
3894 case AArch64::LD2Twov4s_POST:
3895 case AArch64::LD2Twov8b_POST:
3896 case AArch64::LD2Twov8h_POST:
3897 case AArch64::LD2i16_POST:
3898 case AArch64::LD2i32_POST:
3899 case AArch64::LD2i64_POST:
3900 case AArch64::LD2i8_POST:
3901 case AArch64::LD3Rv16b_POST:
3902 case AArch64::LD3Rv1d_POST:
3903 case AArch64::LD3Rv2d_POST:
3904 case AArch64::LD3Rv2s_POST:
3905 case AArch64::LD3Rv4h_POST:
3906 case AArch64::LD3Rv4s_POST:
3907 case AArch64::LD3Rv8b_POST:
3908 case AArch64::LD3Rv8h_POST:
3909 case AArch64::LD3Threev16b_POST:
3910 case AArch64::LD3Threev2d_POST:
3911 case AArch64::LD3Threev2s_POST:
3912 case AArch64::LD3Threev4h_POST:
3913 case AArch64::LD3Threev4s_POST:
3914 case AArch64::LD3Threev8b_POST:
3915 case AArch64::LD3Threev8h_POST:
3916 case AArch64::LD3i16_POST:
3917 case AArch64::LD3i32_POST:
3918 case AArch64::LD3i64_POST:
3919 case AArch64::LD3i8_POST:
3920 case AArch64::LD4Fourv16b_POST:
3921 case AArch64::LD4Fourv2d_POST:
3922 case AArch64::LD4Fourv2s_POST:
3923 case AArch64::LD4Fourv4h_POST:
3924 case AArch64::LD4Fourv4s_POST:
3925 case AArch64::LD4Fourv8b_POST:
3926 case AArch64::LD4Fourv8h_POST:
3927 case AArch64::LD4Rv16b_POST:
3928 case AArch64::LD4Rv1d_POST:
3929 case AArch64::LD4Rv2d_POST:
3930 case AArch64::LD4Rv2s_POST:
3931 case AArch64::LD4Rv4h_POST:
3932 case AArch64::LD4Rv4s_POST:
3933 case AArch64::LD4Rv8b_POST:
3934 case AArch64::LD4Rv8h_POST:
3935 case AArch64::LD4i16_POST:
3936 case AArch64::LD4i32_POST:
3937 case AArch64::LD4i64_POST:
3938 case AArch64::LD4i8_POST:
3939 case AArch64::LDAPRWpost:
3940 case AArch64::LDAPRXpost:
3941 case AArch64::LDIAPPWpost:
3942 case AArch64::LDIAPPXpost:
3943 case AArch64::LDPDpost:
3944 case AArch64::LDPQpost:
3945 case AArch64::LDPSWpost:
3946 case AArch64::LDPSpost:
3947 case AArch64::LDPWpost:
3948 case AArch64::LDPXpost:
3949 case AArch64::LDRBBpost:
3950 case AArch64::LDRBpost:
3951 case AArch64::LDRDpost:
3952 case AArch64::LDRHHpost:
3953 case AArch64::LDRHpost:
3954 case AArch64::LDRQpost:
3955 case AArch64::LDRSBWpost:
3956 case AArch64::LDRSBXpost:
3957 case AArch64::LDRSHWpost:
3958 case AArch64::LDRSHXpost:
3959 case AArch64::LDRSWpost:
3960 case AArch64::LDRSpost:
3961 case AArch64::LDRWpost:
3962 case AArch64::LDRXpost:
3963 case AArch64::ST1Fourv16b_POST:
3964 case AArch64::ST1Fourv1d_POST:
3965 case AArch64::ST1Fourv2d_POST:
3966 case AArch64::ST1Fourv2s_POST:
3967 case AArch64::ST1Fourv4h_POST:
3968 case AArch64::ST1Fourv4s_POST:
3969 case AArch64::ST1Fourv8b_POST:
3970 case AArch64::ST1Fourv8h_POST:
3971 case AArch64::ST1Onev16b_POST:
3972 case AArch64::ST1Onev1d_POST:
3973 case AArch64::ST1Onev2d_POST:
3974 case AArch64::ST1Onev2s_POST:
3975 case AArch64::ST1Onev4h_POST:
3976 case AArch64::ST1Onev4s_POST:
3977 case AArch64::ST1Onev8b_POST:
3978 case AArch64::ST1Onev8h_POST:
3979 case AArch64::ST1Threev16b_POST:
3980 case AArch64::ST1Threev1d_POST:
3981 case AArch64::ST1Threev2d_POST:
3982 case AArch64::ST1Threev2s_POST:
3983 case AArch64::ST1Threev4h_POST:
3984 case AArch64::ST1Threev4s_POST:
3985 case AArch64::ST1Threev8b_POST:
3986 case AArch64::ST1Threev8h_POST:
3987 case AArch64::ST1Twov16b_POST:
3988 case AArch64::ST1Twov1d_POST:
3989 case AArch64::ST1Twov2d_POST:
3990 case AArch64::ST1Twov2s_POST:
3991 case AArch64::ST1Twov4h_POST:
3992 case AArch64::ST1Twov4s_POST:
3993 case AArch64::ST1Twov8b_POST:
3994 case AArch64::ST1Twov8h_POST:
3995 case AArch64::ST1i16_POST:
3996 case AArch64::ST1i32_POST:
3997 case AArch64::ST1i64_POST:
3998 case AArch64::ST1i8_POST:
3999 case AArch64::ST2GPostIndex:
4000 case AArch64::ST2Twov16b_POST:
4001 case AArch64::ST2Twov2d_POST:
4002 case AArch64::ST2Twov2s_POST:
4003 case AArch64::ST2Twov4h_POST:
4004 case AArch64::ST2Twov4s_POST:
4005 case AArch64::ST2Twov8b_POST:
4006 case AArch64::ST2Twov8h_POST:
4007 case AArch64::ST2i16_POST:
4008 case AArch64::ST2i32_POST:
4009 case AArch64::ST2i64_POST:
4010 case AArch64::ST2i8_POST:
4011 case AArch64::ST3Threev16b_POST:
4012 case AArch64::ST3Threev2d_POST:
4013 case AArch64::ST3Threev2s_POST:
4014 case AArch64::ST3Threev4h_POST:
4015 case AArch64::ST3Threev4s_POST:
4016 case AArch64::ST3Threev8b_POST:
4017 case AArch64::ST3Threev8h_POST:
4018 case AArch64::ST3i16_POST:
4019 case AArch64::ST3i32_POST:
4020 case AArch64::ST3i64_POST:
4021 case AArch64::ST3i8_POST:
4022 case AArch64::ST4Fourv16b_POST:
4023 case AArch64::ST4Fourv2d_POST:
4024 case AArch64::ST4Fourv2s_POST:
4025 case AArch64::ST4Fourv4h_POST:
4026 case AArch64::ST4Fourv4s_POST:
4027 case AArch64::ST4Fourv8b_POST:
4028 case AArch64::ST4Fourv8h_POST:
4029 case AArch64::ST4i16_POST:
4030 case AArch64::ST4i32_POST:
4031 case AArch64::ST4i64_POST:
4032 case AArch64::ST4i8_POST:
4033 case AArch64::STGPostIndex:
4034 case AArch64::STGPpost:
4035 case AArch64::STPDpost:
4036 case AArch64::STPQpost:
4037 case AArch64::STPSpost:
4038 case AArch64::STPWpost:
4039 case AArch64::STPXpost:
4040 case AArch64::STRBBpost:
4041 case AArch64::STRBpost:
4042 case AArch64::STRDpost:
4043 case AArch64::STRHHpost:
4044 case AArch64::STRHpost:
4045 case AArch64::STRQpost:
4046 case AArch64::STRSpost:
4047 case AArch64::STRWpost:
4048 case AArch64::STRXpost:
4049 case AArch64::STZ2GPostIndex:
4050 case AArch64::STZGPostIndex:
4057 bool &OffsetIsScalable,
TypeSize &Width,
4078 int64_t Dummy1, Dummy2;
4100 return BaseOp->
isReg() || BaseOp->
isFI();
4107 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
4112 TypeSize &Width, int64_t &MinOffset,
4113 int64_t &MaxOffset) {
4119 MinOffset = MaxOffset = 0;
4122 case AArch64::LDRQui:
4123 case AArch64::STRQui:
4129 case AArch64::LDRXui:
4130 case AArch64::LDRDui:
4131 case AArch64::STRXui:
4132 case AArch64::STRDui:
4133 case AArch64::PRFMui:
4139 case AArch64::LDRWui:
4140 case AArch64::LDRSui:
4141 case AArch64::LDRSWui:
4142 case AArch64::STRWui:
4143 case AArch64::STRSui:
4149 case AArch64::LDRHui:
4150 case AArch64::LDRHHui:
4151 case AArch64::LDRSHWui:
4152 case AArch64::LDRSHXui:
4153 case AArch64::STRHui:
4154 case AArch64::STRHHui:
4160 case AArch64::LDRBui:
4161 case AArch64::LDRBBui:
4162 case AArch64::LDRSBWui:
4163 case AArch64::LDRSBXui:
4164 case AArch64::STRBui:
4165 case AArch64::STRBBui:
4172 case AArch64::STRQpre:
4173 case AArch64::LDRQpost:
4179 case AArch64::LDRDpost:
4180 case AArch64::LDRDpre:
4181 case AArch64::LDRXpost:
4182 case AArch64::LDRXpre:
4183 case AArch64::STRDpost:
4184 case AArch64::STRDpre:
4185 case AArch64::STRXpost:
4186 case AArch64::STRXpre:
4192 case AArch64::STRWpost:
4193 case AArch64::STRWpre:
4194 case AArch64::LDRWpost:
4195 case AArch64::LDRWpre:
4196 case AArch64::STRSpost:
4197 case AArch64::STRSpre:
4198 case AArch64::LDRSpost:
4199 case AArch64::LDRSpre:
4205 case AArch64::LDRHpost:
4206 case AArch64::LDRHpre:
4207 case AArch64::STRHpost:
4208 case AArch64::STRHpre:
4209 case AArch64::LDRHHpost:
4210 case AArch64::LDRHHpre:
4211 case AArch64::STRHHpost:
4212 case AArch64::STRHHpre:
4218 case AArch64::LDRBpost:
4219 case AArch64::LDRBpre:
4220 case AArch64::STRBpost:
4221 case AArch64::STRBpre:
4222 case AArch64::LDRBBpost:
4223 case AArch64::LDRBBpre:
4224 case AArch64::STRBBpost:
4225 case AArch64::STRBBpre:
4232 case AArch64::LDURQi:
4233 case AArch64::STURQi:
4239 case AArch64::LDURXi:
4240 case AArch64::LDURDi:
4241 case AArch64::LDAPURXi:
4242 case AArch64::STURXi:
4243 case AArch64::STURDi:
4244 case AArch64::STLURXi:
4245 case AArch64::PRFUMi:
4251 case AArch64::LDURWi:
4252 case AArch64::LDURSi:
4253 case AArch64::LDURSWi:
4254 case AArch64::LDAPURi:
4255 case AArch64::LDAPURSWi:
4256 case AArch64::STURWi:
4257 case AArch64::STURSi:
4258 case AArch64::STLURWi:
4264 case AArch64::LDURHi:
4265 case AArch64::LDURHHi:
4266 case AArch64::LDURSHXi:
4267 case AArch64::LDURSHWi:
4268 case AArch64::LDAPURHi:
4269 case AArch64::LDAPURSHWi:
4270 case AArch64::LDAPURSHXi:
4271 case AArch64::STURHi:
4272 case AArch64::STURHHi:
4273 case AArch64::STLURHi:
4279 case AArch64::LDURBi:
4280 case AArch64::LDURBBi:
4281 case AArch64::LDURSBXi:
4282 case AArch64::LDURSBWi:
4283 case AArch64::LDAPURBi:
4284 case AArch64::LDAPURSBWi:
4285 case AArch64::LDAPURSBXi:
4286 case AArch64::STURBi:
4287 case AArch64::STURBBi:
4288 case AArch64::STLURBi:
4295 case AArch64::LDPQi:
4296 case AArch64::LDNPQi:
4297 case AArch64::STPQi:
4298 case AArch64::STNPQi:
4299 case AArch64::LDPQpost:
4300 case AArch64::LDPQpre:
4301 case AArch64::STPQpost:
4302 case AArch64::STPQpre:
4308 case AArch64::LDPXi:
4309 case AArch64::LDPDi:
4310 case AArch64::LDNPXi:
4311 case AArch64::LDNPDi:
4312 case AArch64::STPXi:
4313 case AArch64::STPDi:
4314 case AArch64::STNPXi:
4315 case AArch64::STNPDi:
4316 case AArch64::LDPDpost:
4317 case AArch64::LDPDpre:
4318 case AArch64::LDPXpost:
4319 case AArch64::LDPXpre:
4320 case AArch64::STPDpost:
4321 case AArch64::STPDpre:
4322 case AArch64::STPXpost:
4323 case AArch64::STPXpre:
4329 case AArch64::LDPWi:
4330 case AArch64::LDPSi:
4331 case AArch64::LDNPWi:
4332 case AArch64::LDNPSi:
4333 case AArch64::STPWi:
4334 case AArch64::STPSi:
4335 case AArch64::STNPWi:
4336 case AArch64::STNPSi:
4337 case AArch64::LDPSpost:
4338 case AArch64::LDPSpre:
4339 case AArch64::LDPWpost:
4340 case AArch64::LDPWpre:
4341 case AArch64::STPSpost:
4342 case AArch64::STPSpre:
4343 case AArch64::STPWpost:
4344 case AArch64::STPWpre:
4350 case AArch64::StoreSwiftAsyncContext:
4363 case AArch64::TAGPstack:
4373 case AArch64::STGPreIndex:
4374 case AArch64::STGPostIndex:
4375 case AArch64::STZGi:
4376 case AArch64::STZGPreIndex:
4377 case AArch64::STZGPostIndex:
4384 case AArch64::STR_ZZZZXI:
4385 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
4386 case AArch64::LDR_ZZZZXI:
4387 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
4393 case AArch64::STR_ZZZXI:
4394 case AArch64::LDR_ZZZXI:
4400 case AArch64::STR_ZZXI:
4401 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
4402 case AArch64::LDR_ZZXI:
4403 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
4409 case AArch64::LDR_PXI:
4410 case AArch64::STR_PXI:
4416 case AArch64::LDR_PPXI:
4417 case AArch64::STR_PPXI:
4423 case AArch64::LDR_ZXI:
4424 case AArch64::STR_ZXI:
4430 case AArch64::LD1B_IMM:
4431 case AArch64::LD1H_IMM:
4432 case AArch64::LD1W_IMM:
4433 case AArch64::LD1D_IMM:
4434 case AArch64::LDNT1B_ZRI:
4435 case AArch64::LDNT1H_ZRI:
4436 case AArch64::LDNT1W_ZRI:
4437 case AArch64::LDNT1D_ZRI:
4438 case AArch64::ST1B_IMM:
4439 case AArch64::ST1H_IMM:
4440 case AArch64::ST1W_IMM:
4441 case AArch64::ST1D_IMM:
4442 case AArch64::STNT1B_ZRI:
4443 case AArch64::STNT1H_ZRI:
4444 case AArch64::STNT1W_ZRI:
4445 case AArch64::STNT1D_ZRI:
4446 case AArch64::LDNF1B_IMM:
4447 case AArch64::LDNF1H_IMM:
4448 case AArch64::LDNF1W_IMM:
4449 case AArch64::LDNF1D_IMM:
4457 case AArch64::LD2B_IMM:
4458 case AArch64::LD2H_IMM:
4459 case AArch64::LD2W_IMM:
4460 case AArch64::LD2D_IMM:
4461 case AArch64::ST2B_IMM:
4462 case AArch64::ST2H_IMM:
4463 case AArch64::ST2W_IMM:
4464 case AArch64::ST2D_IMM:
4470 case AArch64::LD3B_IMM:
4471 case AArch64::LD3H_IMM:
4472 case AArch64::LD3W_IMM:
4473 case AArch64::LD3D_IMM:
4474 case AArch64::ST3B_IMM:
4475 case AArch64::ST3H_IMM:
4476 case AArch64::ST3W_IMM:
4477 case AArch64::ST3D_IMM:
4483 case AArch64::LD4B_IMM:
4484 case AArch64::LD4H_IMM:
4485 case AArch64::LD4W_IMM:
4486 case AArch64::LD4D_IMM:
4487 case AArch64::ST4B_IMM:
4488 case AArch64::ST4H_IMM:
4489 case AArch64::ST4W_IMM:
4490 case AArch64::ST4D_IMM:
4496 case AArch64::LD1B_H_IMM:
4497 case AArch64::LD1SB_H_IMM:
4498 case AArch64::LD1H_S_IMM:
4499 case AArch64::LD1SH_S_IMM:
4500 case AArch64::LD1W_D_IMM:
4501 case AArch64::LD1SW_D_IMM:
4502 case AArch64::ST1B_H_IMM:
4503 case AArch64::ST1H_S_IMM:
4504 case AArch64::ST1W_D_IMM:
4505 case AArch64::LDNF1B_H_IMM:
4506 case AArch64::LDNF1SB_H_IMM:
4507 case AArch64::LDNF1H_S_IMM:
4508 case AArch64::LDNF1SH_S_IMM:
4509 case AArch64::LDNF1W_D_IMM:
4510 case AArch64::LDNF1SW_D_IMM:
4518 case AArch64::LD1B_S_IMM:
4519 case AArch64::LD1SB_S_IMM:
4520 case AArch64::LD1H_D_IMM:
4521 case AArch64::LD1SH_D_IMM:
4522 case AArch64::ST1B_S_IMM:
4523 case AArch64::ST1H_D_IMM:
4524 case AArch64::LDNF1B_S_IMM:
4525 case AArch64::LDNF1SB_S_IMM:
4526 case AArch64::LDNF1H_D_IMM:
4527 case AArch64::LDNF1SH_D_IMM:
4535 case AArch64::LD1B_D_IMM:
4536 case AArch64::LD1SB_D_IMM:
4537 case AArch64::ST1B_D_IMM:
4538 case AArch64::LDNF1B_D_IMM:
4539 case AArch64::LDNF1SB_D_IMM:
4547 case AArch64::ST2Gi:
4548 case AArch64::ST2GPreIndex:
4549 case AArch64::ST2GPostIndex:
4550 case AArch64::STZ2Gi:
4551 case AArch64::STZ2GPreIndex:
4552 case AArch64::STZ2GPostIndex:
4558 case AArch64::STGPi:
4559 case AArch64::STGPpost:
4560 case AArch64::STGPpre:
4566 case AArch64::LD1RB_IMM:
4567 case AArch64::LD1RB_H_IMM:
4568 case AArch64::LD1RB_S_IMM:
4569 case AArch64::LD1RB_D_IMM:
4570 case AArch64::LD1RSB_H_IMM:
4571 case AArch64::LD1RSB_S_IMM:
4572 case AArch64::LD1RSB_D_IMM:
4578 case AArch64::LD1RH_IMM:
4579 case AArch64::LD1RH_S_IMM:
4580 case AArch64::LD1RH_D_IMM:
4581 case AArch64::LD1RSH_S_IMM:
4582 case AArch64::LD1RSH_D_IMM:
4588 case AArch64::LD1RW_IMM:
4589 case AArch64::LD1RW_D_IMM:
4590 case AArch64::LD1RSW_IMM:
4596 case AArch64::LD1RD_IMM:
4612 case AArch64::LDRBBui:
4613 case AArch64::LDURBBi:
4614 case AArch64::LDRSBWui:
4615 case AArch64::LDURSBWi:
4616 case AArch64::STRBBui:
4617 case AArch64::STURBBi:
4619 case AArch64::LDRHHui:
4620 case AArch64::LDURHHi:
4621 case AArch64::LDRSHWui:
4622 case AArch64::LDURSHWi:
4623 case AArch64::STRHHui:
4624 case AArch64::STURHHi:
4626 case AArch64::LDRSui:
4627 case AArch64::LDURSi:
4628 case AArch64::LDRSpre:
4629 case AArch64::LDRSWui:
4630 case AArch64::LDURSWi:
4631 case AArch64::LDRSWpre:
4632 case AArch64::LDRWpre:
4633 case AArch64::LDRWui:
4634 case AArch64::LDURWi:
4635 case AArch64::STRSui:
4636 case AArch64::STURSi:
4637 case AArch64::STRSpre:
4638 case AArch64::STRWui:
4639 case AArch64::STURWi:
4640 case AArch64::STRWpre:
4641 case AArch64::LDPSi:
4642 case AArch64::LDPSWi:
4643 case AArch64::LDPWi:
4644 case AArch64::STPSi:
4645 case AArch64::STPWi:
4647 case AArch64::LDRDui:
4648 case AArch64::LDURDi:
4649 case AArch64::LDRDpre:
4650 case AArch64::LDRXui:
4651 case AArch64::LDURXi:
4652 case AArch64::LDRXpre:
4653 case AArch64::STRDui:
4654 case AArch64::STURDi:
4655 case AArch64::STRDpre:
4656 case AArch64::STRXui:
4657 case AArch64::STURXi:
4658 case AArch64::STRXpre:
4659 case AArch64::LDPDi:
4660 case AArch64::LDPXi:
4661 case AArch64::STPDi:
4662 case AArch64::STPXi:
4664 case AArch64::LDRQui:
4665 case AArch64::LDURQi:
4666 case AArch64::STRQui:
4667 case AArch64::STURQi:
4668 case AArch64::STRQpre:
4669 case AArch64::LDPQi:
4670 case AArch64::LDRQpre:
4671 case AArch64::STPQi:
4673 case AArch64::STZGi:
4674 case AArch64::ST2Gi:
4675 case AArch64::STZ2Gi:
4676 case AArch64::STGPi:
4682 switch (
MI.getOpcode()) {
4685 case AArch64::LDRWpre:
4686 case AArch64::LDRXpre:
4687 case AArch64::LDRSWpre:
4688 case AArch64::LDRSpre:
4689 case AArch64::LDRDpre:
4690 case AArch64::LDRQpre:
4696 switch (
MI.getOpcode()) {
4699 case AArch64::STRWpre:
4700 case AArch64::STRXpre:
4701 case AArch64::STRSpre:
4702 case AArch64::STRDpre:
4703 case AArch64::STRQpre:
4713 switch (
MI.getOpcode()) {
4716 case AArch64::LDPSi:
4717 case AArch64::LDPSWi:
4718 case AArch64::LDPDi:
4719 case AArch64::LDPQi:
4720 case AArch64::LDPWi:
4721 case AArch64::LDPXi:
4722 case AArch64::STPSi:
4723 case AArch64::STPDi:
4724 case AArch64::STPQi:
4725 case AArch64::STPWi:
4726 case AArch64::STPXi:
4727 case AArch64::STGPi:
4733 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4737 return MI.getOperand(Idx);
4742 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4746 return MI.getOperand(Idx);
4751 switch (
MI.getOpcode()) {
4754 case AArch64::LDRBroX:
4755 case AArch64::LDRBBroX:
4756 case AArch64::LDRSBXroX:
4757 case AArch64::LDRSBWroX:
4758 case AArch64::LDRHroX:
4759 case AArch64::LDRHHroX:
4760 case AArch64::LDRSHXroX:
4761 case AArch64::LDRSHWroX:
4762 case AArch64::LDRWroX:
4763 case AArch64::LDRSroX:
4764 case AArch64::LDRSWroX:
4765 case AArch64::LDRDroX:
4766 case AArch64::LDRXroX:
4767 case AArch64::LDRQroX:
4768 return MI.getOperand(4);
4774 if (
MI.getParent() ==
nullptr)
4784 auto Reg =
Op.getReg();
4785 if (Reg.isPhysical())
4786 return AArch64::FPR16RegClass.contains(Reg);
4788 return TRC == &AArch64::FPR16RegClass ||
4789 TRC == &AArch64::FPR16_loRegClass;
4798 auto Reg =
Op.getReg();
4799 if (Reg.isPhysical())
4800 return AArch64::FPR128RegClass.contains(Reg);
4802 return TRC == &AArch64::FPR128RegClass ||
4803 TRC == &AArch64::FPR128_loRegClass;
4809 switch (
MI.getOpcode()) {
4812 case AArch64::PACIASP:
4813 case AArch64::PACIBSP:
4816 case AArch64::PAUTH_PROLOGUE:
4819 case AArch64::HINT: {
4820 unsigned Imm =
MI.getOperand(0).getImm();
4822 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4825 if (Imm == 25 || Imm == 27)
4837 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4838 return AArch64::FPR128RegClass.contains(Reg) ||
4839 AArch64::FPR64RegClass.contains(Reg) ||
4840 AArch64::FPR32RegClass.contains(Reg) ||
4841 AArch64::FPR16RegClass.contains(Reg) ||
4842 AArch64::FPR8RegClass.contains(Reg);
4849 auto Reg =
Op.getReg();
4850 if (Reg.isPhysical())
4854 return TRC == &AArch64::FPR128RegClass ||
4855 TRC == &AArch64::FPR128_loRegClass ||
4856 TRC == &AArch64::FPR64RegClass ||
4857 TRC == &AArch64::FPR64_loRegClass ||
4858 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4859 TRC == &AArch64::FPR8RegClass;
4881 if (FirstOpc == SecondOpc)
4887 case AArch64::STRSui:
4888 case AArch64::STURSi:
4889 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4890 case AArch64::STRDui:
4891 case AArch64::STURDi:
4892 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4893 case AArch64::STRQui:
4894 case AArch64::STURQi:
4895 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4896 case AArch64::STRWui:
4897 case AArch64::STURWi:
4898 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4899 case AArch64::STRXui:
4900 case AArch64::STURXi:
4901 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4902 case AArch64::LDRSui:
4903 case AArch64::LDURSi:
4904 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4905 case AArch64::LDRDui:
4906 case AArch64::LDURDi:
4907 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4908 case AArch64::LDRQui:
4909 case AArch64::LDURQi:
4910 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4911 case AArch64::LDRWui:
4912 case AArch64::LDURWi:
4913 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4914 case AArch64::LDRSWui:
4915 case AArch64::LDURSWi:
4916 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4917 case AArch64::LDRXui:
4918 case AArch64::LDURXi:
4919 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4926 int64_t Offset1,
unsigned Opcode1,
int FI2,
4927 int64_t Offset2,
unsigned Opcode2) {
4933 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4936 if (ObjectOffset1 % Scale1 != 0)
4938 ObjectOffset1 /= Scale1;
4940 if (ObjectOffset2 % Scale2 != 0)
4942 ObjectOffset2 /= Scale2;
4943 ObjectOffset1 += Offset1;
4944 ObjectOffset2 += Offset2;
4945 return ObjectOffset1 + 1 == ObjectOffset2;
4957 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4958 unsigned NumBytes)
const {
4968 "Only base registers and frame indices are supported.");
4975 if (ClusterSize > 2)
4982 unsigned FirstOpc = FirstLdSt.
getOpcode();
4983 unsigned SecondOpc = SecondLdSt.
getOpcode();
5003 if (Offset1 > 63 || Offset1 < -64)
5008 if (BaseOp1.
isFI()) {
5010 "Caller should have ordered offsets.");
5015 BaseOp2.
getIndex(), Offset2, SecondOpc);
5018 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
5020 return Offset1 + 1 == Offset2;
5030 if (
Reg.isPhysical())
5039 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
5048 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
5050 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
5051 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
5052 unsigned NumRegs = Indices.
size();
5054 int SubReg = 0, End = NumRegs, Incr = 1;
5073 unsigned Opcode,
unsigned ZeroReg,
5076 unsigned NumRegs = Indices.
size();
5079 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
5080 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
5081 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5082 "GPR reg sequences should not be able to overlap");
5099 bool RenamableSrc)
const {
5100 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
5101 AArch64::GPR32spRegClass.
contains(SrcReg)) {
5102 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5104 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5105 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5107 MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
5108 &AArch64::GPR64spRegClass);
5109 MCRegister SrcRegX = RI.getMatchingSuperReg(SrcReg, AArch64::sub_32,
5110 &AArch64::GPR64spRegClass);
5126 }
else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5127 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5129 MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
5130 &AArch64::GPR64spRegClass);
5131 assert(DestRegX.
isValid() &&
"Destination super-reg not valid");
5132 MCRegister SrcRegX = RI.getMatchingSuperReg(SrcReg, AArch64::sub_32,
5133 &AArch64::GPR64spRegClass);
5153 if (AArch64::GPR32spRegClass.
contains(DestReg) && SrcReg == AArch64::WZR) {
5154 if (Subtarget.hasZeroCycleZeroingGPR32()) {
5166 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5167 AArch64::GPR64spRegClass.
contains(SrcReg)) {
5168 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5184 if (AArch64::GPR64spRegClass.
contains(DestReg) && SrcReg == AArch64::XZR) {
5185 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5198 if (AArch64::PPRRegClass.
contains(DestReg) &&
5199 AArch64::PPRRegClass.
contains(SrcReg)) {
5200 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5201 "Unexpected SVE register.");
5211 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
5212 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
5213 if (DestIsPNR || SrcIsPNR) {
5215 return (R - AArch64::PN0) + AArch64::P0;
5220 if (PPRSrcReg != PPRDestReg) {
5232 if (AArch64::ZPRRegClass.
contains(DestReg) &&
5233 AArch64::ZPRRegClass.
contains(SrcReg)) {
5234 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5235 "Unexpected SVE register.");
5243 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
5244 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
5245 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
5246 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
5247 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5248 "Unexpected SVE register.");
5249 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5256 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
5257 AArch64::ZPR3RegClass.
contains(SrcReg)) {
5258 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5259 "Unexpected SVE register.");
5260 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5268 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
5269 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
5270 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
5271 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5272 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5273 "Unexpected SVE register.");
5274 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5275 AArch64::zsub2, AArch64::zsub3};
5282 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5283 AArch64::DDDDRegClass.
contains(SrcReg)) {
5284 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5285 AArch64::dsub2, AArch64::dsub3};
5292 if (AArch64::DDDRegClass.
contains(DestReg) &&
5293 AArch64::DDDRegClass.
contains(SrcReg)) {
5294 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5302 if (AArch64::DDRegClass.
contains(DestReg) &&
5303 AArch64::DDRegClass.
contains(SrcReg)) {
5304 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5311 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5312 AArch64::QQQQRegClass.
contains(SrcReg)) {
5313 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5314 AArch64::qsub2, AArch64::qsub3};
5321 if (AArch64::QQQRegClass.
contains(DestReg) &&
5322 AArch64::QQQRegClass.
contains(SrcReg)) {
5323 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5331 if (AArch64::QQRegClass.
contains(DestReg) &&
5332 AArch64::QQRegClass.
contains(SrcReg)) {
5333 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5339 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5340 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5341 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5343 AArch64::XZR, Indices);
5347 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5348 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5349 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5351 AArch64::WZR, Indices);
5355 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5356 AArch64::FPR128RegClass.
contains(SrcReg)) {
5357 if (Subtarget.isSVEorStreamingSVEAvailable() &&
5358 !Subtarget.isNeonAvailable())
5361 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5362 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5363 else if (Subtarget.isNeonAvailable())
5382 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5383 AArch64::FPR64RegClass.
contains(SrcReg)) {
5384 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5385 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5386 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5387 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
5388 &AArch64::FPR128RegClass);
5389 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
5390 &AArch64::FPR128RegClass);
5406 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5407 AArch64::FPR32RegClass.
contains(SrcReg)) {
5408 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5409 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5410 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5411 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
5412 &AArch64::FPR128RegClass);
5413 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
5414 &AArch64::FPR128RegClass);
5423 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5424 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5425 MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
5426 &AArch64::FPR64RegClass);
5427 MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
5428 &AArch64::FPR64RegClass);
5443 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5444 AArch64::FPR16RegClass.
contains(SrcReg)) {
5445 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5446 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5447 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5448 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5449 &AArch64::FPR128RegClass);
5450 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5451 &AArch64::FPR128RegClass);
5460 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5461 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5462 MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5463 &AArch64::FPR64RegClass);
5464 MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5465 &AArch64::FPR64RegClass);
5474 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5475 &AArch64::FPR32RegClass);
5476 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5477 &AArch64::FPR32RegClass);
5484 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5485 AArch64::FPR8RegClass.
contains(SrcReg)) {
5486 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5487 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5488 !Subtarget.hasZeroCycleRegMoveFPR64() && Subtarget.isNeonAvailable()) {
5489 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5490 &AArch64::FPR128RegClass);
5491 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5492 &AArch64::FPR128RegClass);
5501 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5502 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5503 MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5504 &AArch64::FPR64RegClass);
5505 MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5506 &AArch64::FPR64RegClass);
5515 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5516 &AArch64::FPR32RegClass);
5517 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5518 &AArch64::FPR32RegClass);
5526 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5527 AArch64::GPR64RegClass.
contains(SrcReg)) {
5528 if (AArch64::XZR == SrcReg) {
5536 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5537 AArch64::FPR64RegClass.
contains(SrcReg)) {
5543 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5544 AArch64::GPR32RegClass.
contains(SrcReg)) {
5545 if (AArch64::WZR == SrcReg) {
5553 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5554 AArch64::FPR32RegClass.
contains(SrcReg)) {
5560 if (DestReg == AArch64::NZCV) {
5561 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5563 .
addImm(AArch64SysReg::NZCV)
5569 if (SrcReg == AArch64::NZCV) {
5570 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
5572 .
addImm(AArch64SysReg::NZCV)
5578 errs() << RI.getRegAsmName(DestReg) <<
" = COPY " << RI.getRegAsmName(SrcReg)
5589 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
5594 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
5596 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
5609 Register SrcReg,
bool isKill,
int FI,
5625 switch (
TRI->getSpillSize(*RC)) {
5627 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5628 Opc = AArch64::STRBui;
5631 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5632 Opc = AArch64::STRHui;
5633 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
5634 AArch64::PPRRegClass.hasSubClassEq(RC)) {
5635 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5636 "Unexpected register store without SVE store instructions");
5637 Opc = AArch64::STR_PXI;
5643 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5644 Opc = AArch64::STRWui;
5648 assert(SrcReg != AArch64::WSP);
5649 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5650 Opc = AArch64::STRSui;
5651 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5652 Opc = AArch64::STR_PPXI;
5657 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5658 Opc = AArch64::STRXui;
5662 assert(SrcReg != AArch64::SP);
5663 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5664 Opc = AArch64::STRDui;
5665 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5667 get(AArch64::STPWi), SrcReg, isKill,
5668 AArch64::sube32, AArch64::subo32, FI, MMO);
5673 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5674 Opc = AArch64::STRQui;
5675 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5676 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5677 Opc = AArch64::ST1Twov1d;
5679 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5681 get(AArch64::STPXi), SrcReg, isKill,
5682 AArch64::sube64, AArch64::subo64, FI, MMO);
5684 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5685 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5686 "Unexpected register store without SVE store instructions");
5687 Opc = AArch64::STR_ZXI;
5692 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5693 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5694 Opc = AArch64::ST1Threev1d;
5699 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5700 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5701 Opc = AArch64::ST1Fourv1d;
5703 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5704 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5705 Opc = AArch64::ST1Twov2d;
5707 }
else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5708 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5709 "Unexpected register store without SVE store instructions");
5710 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
5712 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
5713 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5714 "Unexpected register store without SVE store instructions");
5715 Opc = AArch64::STR_ZZXI;
5720 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5721 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5722 Opc = AArch64::ST1Threev2d;
5724 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5725 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5726 "Unexpected register store without SVE store instructions");
5727 Opc = AArch64::STR_ZZZXI;
5732 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5733 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5734 Opc = AArch64::ST1Fourv2d;
5736 }
else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5737 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5738 "Unexpected register store without SVE store instructions");
5739 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
5741 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
5742 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5743 "Unexpected register store without SVE store instructions");
5744 Opc = AArch64::STR_ZZZZXI;
5749 assert(
Opc &&
"Unknown register class");
5760 MI.addMemOperand(MMO);
5767 Register DestReg,
unsigned SubIdx0,
5768 unsigned SubIdx1,
int FI,
5772 bool IsUndef =
true;
5774 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
5776 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5803 switch (
TRI->getSpillSize(*RC)) {
5805 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5806 Opc = AArch64::LDRBui;
5809 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5810 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5811 Opc = AArch64::LDRHui;
5812 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5813 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5814 "Unexpected register load without SVE load instructions");
5817 Opc = AArch64::LDR_PXI;
5823 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5824 Opc = AArch64::LDRWui;
5828 assert(DestReg != AArch64::WSP);
5829 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5830 Opc = AArch64::LDRSui;
5831 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5832 Opc = AArch64::LDR_PPXI;
5837 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5838 Opc = AArch64::LDRXui;
5842 assert(DestReg != AArch64::SP);
5843 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5844 Opc = AArch64::LDRDui;
5845 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5847 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5848 AArch64::subo32, FI, MMO);
5853 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5854 Opc = AArch64::LDRQui;
5855 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5856 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5857 Opc = AArch64::LD1Twov1d;
5859 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5861 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5862 AArch64::subo64, FI, MMO);
5864 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5865 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5866 "Unexpected register load without SVE load instructions");
5867 Opc = AArch64::LDR_ZXI;
5872 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5873 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5874 Opc = AArch64::LD1Threev1d;
5879 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5880 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5881 Opc = AArch64::LD1Fourv1d;
5883 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5884 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5885 Opc = AArch64::LD1Twov2d;
5887 }
else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5888 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5889 "Unexpected register load without SVE load instructions");
5890 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
5892 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
5893 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5894 "Unexpected register load without SVE load instructions");
5895 Opc = AArch64::LDR_ZZXI;
5900 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5901 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5902 Opc = AArch64::LD1Threev2d;
5904 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5905 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5906 "Unexpected register load without SVE load instructions");
5907 Opc = AArch64::LDR_ZZZXI;
5912 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5913 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5914 Opc = AArch64::LD1Fourv2d;
5916 }
else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5917 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5918 "Unexpected register load without SVE load instructions");
5919 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
5921 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
5922 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5923 "Unexpected register load without SVE load instructions");
5924 Opc = AArch64::LDR_ZZZZXI;
5930 assert(
Opc &&
"Unknown register class");
5940 MI.addMemOperand(MMO);
5947 UseMI.getIterator()),
5949 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5950 I.readsRegister(AArch64::NZCV, TRI);
5954void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
5959 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5966 ByteSized =
Offset.getFixed();
5967 VGSized =
Offset.getScalable() / 2;
5973void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
5975 int64_t &NumDataVectors) {
5979 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5981 NumBytes =
Offset.getFixed();
5983 NumPredicateVectors =
Offset.getScalable() / 2;
5988 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5989 NumPredicateVectors > 62) {
5990 NumDataVectors = NumPredicateVectors / 8;
5991 NumPredicateVectors -= NumDataVectors * 8;
6017 Expr.
push_back((
char)dwarf::DW_OP_bregx);
6025 int64_t OffsetFromDefCFA) {
6039 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
6040 if (!RegScale.empty())
6050 int64_t NumBytes, NumVGScaledBytes;
6051 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
Offset, NumBytes,
6053 std::string CommentBuffer;
6056 if (
Reg == AArch64::SP)
6058 else if (
Reg == AArch64::FP)
6065 unsigned DwarfReg =
TRI.getDwarfRegNum(
Reg,
true);
6066 assert(DwarfReg <= 31 &&
"DwarfReg out of bounds (0..31)");
6068 Expr.
push_back(dwarf::DW_OP_breg0 + DwarfReg);
6071 if (NumVGScaledBytes) {
6081 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
6089 unsigned FrameReg,
unsigned Reg,
6091 bool LastAdjustmentWasScalable) {
6092 if (
Offset.getScalable())
6095 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6098 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
6105 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6106 int64_t NumBytes, NumVGScaledBytes;
6107 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6108 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
6110 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
6113 if (!NumVGScaledBytes)
6116 std::string CommentBuffer;
6121 assert(NumVGScaledBytes &&
"Expected scalable offset");
6125 if (IncomingVGOffsetFromDefCFA) {
6127 VGRegScale =
"* IncomingVG";
6130 VGRegScale =
"* VG";
6134 OffsetExpr.
push_back(dwarf::DW_OP_plus);
6143 CfaExpr.
push_back(dwarf::DW_CFA_expression);
6158 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
6161 bool *HasWinCFI,
bool EmitCFAOffset,
6164 unsigned MaxEncoding, ShiftSize;
6166 case AArch64::ADDXri:
6167 case AArch64::ADDSXri:
6168 case AArch64::SUBXri:
6169 case AArch64::SUBSXri:
6170 MaxEncoding = 0xfff;
6173 case AArch64::ADDVL_XXI:
6174 case AArch64::ADDPL_XXI:
6175 case AArch64::ADDSVL_XXI:
6176 case AArch64::ADDSPL_XXI:
6191 if (
Opc == AArch64::ADDVL_XXI ||
Opc == AArch64::ADDSVL_XXI)
6193 else if (
Opc == AArch64::ADDPL_XXI ||
Opc == AArch64::ADDSPL_XXI)
6207 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
6209 if (TmpReg == AArch64::XZR)
6210 TmpReg =
MBB.getParent()->getRegInfo().createVirtualRegister(
6211 &AArch64::GPR64RegClass);
6213 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
6214 unsigned LocalShiftSize = 0;
6215 if (ThisVal > MaxEncoding) {
6216 ThisVal = ThisVal >> ShiftSize;
6217 LocalShiftSize = ShiftSize;
6219 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
6220 "Encoding cannot handle value that big");
6222 Offset -= ThisVal << LocalShiftSize;
6227 .
addImm(Sign * (
int)ThisVal);
6237 if (Sign == -1 ||
Opc == AArch64::SUBXri ||
Opc == AArch64::SUBSXri)
6238 CFAOffset += Change;
6240 CFAOffset -= Change;
6241 if (EmitCFAOffset && DestReg == TmpReg) {
6254 int Imm = (int)(ThisVal << LocalShiftSize);
6255 if (VScale != 1 && DestReg == AArch64::SP) {
6261 }
else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
6262 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
6263 assert(VScale == 1 &&
"Expected non-scalable operation");
6272 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
6273 "emit a single SEH directive");
6274 }
else if (DestReg == AArch64::SP) {
6275 assert(VScale == 1 &&
"Expected non-scalable operation");
6278 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
6291 unsigned DestReg,
unsigned SrcReg,
6294 bool NeedsWinCFI,
bool *HasWinCFI,
6296 unsigned FrameReg) {
6303 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
6305 int64_t Bytes, NumPredicateVectors, NumDataVectors;
6306 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6307 Offset, Bytes, NumPredicateVectors, NumDataVectors);
6310 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
6311 if (NeedsFinalDefNZCV)
6315 if (Bytes || (!
Offset && SrcReg != DestReg)) {
6316 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
6317 "SP increment/decrement not 8-byte aligned");
6318 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
6321 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
6324 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6326 CFAOffset += (
Opc == AArch64::ADDXri ||
Opc == AArch64::ADDSXri)
6333 assert(!(NeedsWinCFI && NumPredicateVectors) &&
6334 "WinCFI can't allocate fractions of an SVE data vector");
6336 if (NumDataVectors) {
6338 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
TII,
6339 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6345 if (NumPredicateVectors) {
6346 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
6348 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
TII,
6349 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6353 if (NeedsFinalDefNZCV)
6374 if (
MI.isFullCopy()) {
6377 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
6381 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
6386 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
6414 if (
MI.isCopy() &&
Ops.size() == 1 &&
6416 (
Ops[0] == 0 ||
Ops[0] == 1)) {
6417 bool IsSpill =
Ops[0] == 0;
6418 bool IsFill = !IsSpill;
6430 :
TRI.getMinimalPhysRegClass(Reg);
6436 "Mismatched register size in non subreg COPY");
6443 return &*--InsertPt;
6455 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
6458 "Unexpected subreg on physical register");
6460 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
6462 return &*--InsertPt;
6479 case AArch64::sub_32:
6480 if (AArch64::GPR64RegClass.hasSubClassEq(
getRegClass(DstReg)))
6481 FillRC = &AArch64::GPR32RegClass;
6484 FillRC = &AArch64::FPR32RegClass;
6487 FillRC = &AArch64::FPR64RegClass;
6493 TRI.getRegSizeInBits(*FillRC) &&
6494 "Mismatched regclass size on folded subreg COPY");
6513 bool *OutUseUnscaledOp,
6514 unsigned *OutUnscaledOp,
6515 int64_t *EmittableOffset) {
6517 if (EmittableOffset)
6518 *EmittableOffset = 0;
6519 if (OutUseUnscaledOp)
6520 *OutUseUnscaledOp =
false;
6526 switch (
MI.getOpcode()) {
6529 case AArch64::LD1Rv1d:
6530 case AArch64::LD1Rv2s:
6531 case AArch64::LD1Rv2d:
6532 case AArch64::LD1Rv4h:
6533 case AArch64::LD1Rv4s:
6534 case AArch64::LD1Rv8b:
6535 case AArch64::LD1Rv8h:
6536 case AArch64::LD1Rv16b:
6537 case AArch64::LD1Twov2d:
6538 case AArch64::LD1Threev2d:
6539 case AArch64::LD1Fourv2d:
6540 case AArch64::LD1Twov1d:
6541 case AArch64::LD1Threev1d:
6542 case AArch64::LD1Fourv1d:
6543 case AArch64::ST1Twov2d:
6544 case AArch64::ST1Threev2d:
6545 case AArch64::ST1Fourv2d:
6546 case AArch64::ST1Twov1d:
6547 case AArch64::ST1Threev1d:
6548 case AArch64::ST1Fourv1d:
6549 case AArch64::ST1i8:
6550 case AArch64::ST1i16:
6551 case AArch64::ST1i32:
6552 case AArch64::ST1i64:
6554 case AArch64::IRGstack:
6555 case AArch64::STGloop:
6556 case AArch64::STZGloop:
6561 TypeSize ScaleValue(0U,
false), Width(0U,
false);
6562 int64_t MinOff, MaxOff;
6568 bool IsMulVL = ScaleValue.isScalable();
6569 unsigned Scale = ScaleValue.getKnownMinValue();
6579 std::optional<unsigned> UnscaledOp =
6581 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
6582 if (useUnscaledOp &&
6587 Scale = ScaleValue.getKnownMinValue();
6588 assert(IsMulVL == ScaleValue.isScalable() &&
6589 "Unscaled opcode has different value for scalable");
6591 int64_t Remainder =
Offset % Scale;
6592 assert(!(Remainder && useUnscaledOp) &&
6593 "Cannot have remainder when using unscaled op");
6595 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
6596 int64_t NewOffset =
Offset / Scale;
6597 if (MinOff <= NewOffset && NewOffset <= MaxOff)
6600 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
6604 if (EmittableOffset)
6605 *EmittableOffset = NewOffset;
6606 if (OutUseUnscaledOp)
6607 *OutUseUnscaledOp = useUnscaledOp;
6608 if (OutUnscaledOp && UnscaledOp)
6609 *OutUnscaledOp = *UnscaledOp;
6622 unsigned Opcode =
MI.getOpcode();
6623 unsigned ImmIdx = FrameRegIdx + 1;
6625 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
6630 MI.eraseFromParent();
6636 unsigned UnscaledOp;
6639 &UnscaledOp, &NewOffset);
6643 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
6645 MI.setDesc(
TII->get(UnscaledOp));
6647 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
6665bool AArch64InstrInfo::useMachineCombiner()
const {
return true; }
6670 case AArch64::ADDSWrr:
6671 case AArch64::ADDSWri:
6672 case AArch64::ADDSXrr:
6673 case AArch64::ADDSXri:
6674 case AArch64::SUBSWrr:
6675 case AArch64::SUBSXrr:
6677 case AArch64::SUBSWri:
6678 case AArch64::SUBSXri:
6689 case AArch64::ADDWrr:
6690 case AArch64::ADDWri:
6691 case AArch64::SUBWrr:
6692 case AArch64::ADDSWrr:
6693 case AArch64::ADDSWri:
6694 case AArch64::SUBSWrr:
6696 case AArch64::SUBWri:
6697 case AArch64::SUBSWri:
6708 case AArch64::ADDXrr:
6709 case AArch64::ADDXri:
6710 case AArch64::SUBXrr:
6711 case AArch64::ADDSXrr:
6712 case AArch64::ADDSXri:
6713 case AArch64::SUBSXrr:
6715 case AArch64::SUBXri:
6716 case AArch64::SUBSXri:
6717 case AArch64::ADDv8i8:
6718 case AArch64::ADDv16i8:
6719 case AArch64::ADDv4i16:
6720 case AArch64::ADDv8i16:
6721 case AArch64::ADDv2i32:
6722 case AArch64::ADDv4i32:
6723 case AArch64::SUBv8i8:
6724 case AArch64::SUBv16i8:
6725 case AArch64::SUBv4i16:
6726 case AArch64::SUBv8i16:
6727 case AArch64::SUBv2i32:
6728 case AArch64::SUBv4i32:
6741 case AArch64::FADDHrr:
6742 case AArch64::FADDSrr:
6743 case AArch64::FADDDrr:
6744 case AArch64::FADDv4f16:
6745 case AArch64::FADDv8f16:
6746 case AArch64::FADDv2f32:
6747 case AArch64::FADDv2f64:
6748 case AArch64::FADDv4f32:
6749 case AArch64::FSUBHrr:
6750 case AArch64::FSUBSrr:
6751 case AArch64::FSUBDrr:
6752 case AArch64::FSUBv4f16:
6753 case AArch64::FSUBv8f16:
6754 case AArch64::FSUBv2f32:
6755 case AArch64::FSUBv2f64:
6756 case AArch64::FSUBv4f32:
6775 unsigned CombineOpc,
unsigned ZeroReg = 0,
6776 bool CheckZeroReg =
false) {
6783 if (!
MI ||
MI->getParent() != &
MBB ||
MI->getOpcode() != CombineOpc)
6786 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
6790 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
6791 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
6792 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
6794 if (
MI->getOperand(3).getReg() != ZeroReg)
6799 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
6808 unsigned MulOpc,
unsigned ZeroReg) {
6823bool AArch64InstrInfo::isAssociativeAndCommutative(
const MachineInstr &Inst,
6824 bool Invert)
const {
6830 case AArch64::FADDHrr:
6831 case AArch64::FADDSrr:
6832 case AArch64::FADDDrr:
6833 case AArch64::FMULHrr:
6834 case AArch64::FMULSrr:
6835 case AArch64::FMULDrr:
6836 case AArch64::FMULX16:
6837 case AArch64::FMULX32:
6838 case AArch64::FMULX64:
6840 case AArch64::FADDv4f16:
6841 case AArch64::FADDv8f16:
6842 case AArch64::FADDv2f32:
6843 case AArch64::FADDv4f32:
6844 case AArch64::FADDv2f64:
6845 case AArch64::FMULv4f16:
6846 case AArch64::FMULv8f16:
6847 case AArch64::FMULv2f32:
6848 case AArch64::FMULv4f32:
6849 case AArch64::FMULv2f64:
6850 case AArch64::FMULXv4f16:
6851 case AArch64::FMULXv8f16:
6852 case AArch64::FMULXv2f32:
6853 case AArch64::FMULXv4f32:
6854 case AArch64::FMULXv2f64:
6858 case AArch64::FADD_ZZZ_H:
6859 case AArch64::FADD_ZZZ_S:
6860 case AArch64::FADD_ZZZ_D:
6861 case AArch64::FMUL_ZZZ_H:
6862 case AArch64::FMUL_ZZZ_S:
6863 case AArch64::FMUL_ZZZ_D:
6874 case AArch64::ADDWrr:
6875 case AArch64::ADDXrr:
6876 case AArch64::ANDWrr:
6877 case AArch64::ANDXrr:
6878 case AArch64::ORRWrr:
6879 case AArch64::ORRXrr:
6880 case AArch64::EORWrr:
6881 case AArch64::EORXrr:
6882 case AArch64::EONWrr:
6883 case AArch64::EONXrr:
6887 case AArch64::ADDv8i8:
6888 case AArch64::ADDv16i8:
6889 case AArch64::ADDv4i16:
6890 case AArch64::ADDv8i16:
6891 case AArch64::ADDv2i32:
6892 case AArch64::ADDv4i32:
6893 case AArch64::ADDv1i64:
6894 case AArch64::ADDv2i64:
6895 case AArch64::MULv8i8:
6896 case AArch64::MULv16i8:
6897 case AArch64::MULv4i16:
6898 case AArch64::MULv8i16:
6899 case AArch64::MULv2i32:
6900 case AArch64::MULv4i32:
6901 case AArch64::ANDv8i8:
6902 case AArch64::ANDv16i8:
6903 case AArch64::ORRv8i8:
6904 case AArch64::ORRv16i8:
6905 case AArch64::EORv8i8:
6906 case AArch64::EORv16i8:
6908 case AArch64::ADD_ZZZ_B:
6909 case AArch64::ADD_ZZZ_H:
6910 case AArch64::ADD_ZZZ_S:
6911 case AArch64::ADD_ZZZ_D:
6912 case AArch64::MUL_ZZZ_B:
6913 case AArch64::MUL_ZZZ_H:
6914 case AArch64::MUL_ZZZ_S:
6915 case AArch64::MUL_ZZZ_D:
6916 case AArch64::AND_ZZZ:
6917 case AArch64::ORR_ZZZ:
6918 case AArch64::EOR_ZZZ:
6949 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6957 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6969 case AArch64::ADDWrr:
6971 "ADDWrr does not have register operands");
6972 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6973 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6975 case AArch64::ADDXrr:
6976 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6977 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6979 case AArch64::SUBWrr:
6980 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6981 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6983 case AArch64::SUBXrr:
6984 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6985 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6987 case AArch64::ADDWri:
6988 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6990 case AArch64::ADDXri:
6991 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6993 case AArch64::SUBWri:
6994 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6996 case AArch64::SUBXri:
6997 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6999 case AArch64::ADDv8i8:
7000 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7001 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7003 case AArch64::ADDv16i8:
7004 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7005 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7007 case AArch64::ADDv4i16:
7008 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7009 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7010 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7011 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7013 case AArch64::ADDv8i16:
7014 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7015 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7016 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7017 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7019 case AArch64::ADDv2i32:
7020 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7021 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7022 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7023 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7025 case AArch64::ADDv4i32:
7026 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7027 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7028 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7029 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7031 case AArch64::SUBv8i8:
7032 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7033 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7035 case AArch64::SUBv16i8:
7036 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7037 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7039 case AArch64::SUBv4i16:
7040 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7041 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7042 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7043 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7045 case AArch64::SUBv8i16:
7046 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7047 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7048 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7049 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7051 case AArch64::SUBv2i32:
7052 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7053 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7054 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7055 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7057 case AArch64::SUBv4i32:
7058 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7059 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7060 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7061 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7067bool AArch64InstrInfo::isAccumulationOpcode(
unsigned Opcode)
const {
7071 case AArch64::UABALB_ZZZ_D:
7072 case AArch64::UABALB_ZZZ_H:
7073 case AArch64::UABALB_ZZZ_S:
7074 case AArch64::UABALT_ZZZ_D:
7075 case AArch64::UABALT_ZZZ_H:
7076 case AArch64::UABALT_ZZZ_S:
7077 case AArch64::SABALB_ZZZ_D:
7078 case AArch64::SABALB_ZZZ_S:
7079 case AArch64::SABALB_ZZZ_H:
7080 case AArch64::SABALT_ZZZ_D:
7081 case AArch64::SABALT_ZZZ_S:
7082 case AArch64::SABALT_ZZZ_H:
7083 case AArch64::UABALv16i8_v8i16:
7084 case AArch64::UABALv2i32_v2i64:
7085 case AArch64::UABALv4i16_v4i32:
7086 case AArch64::UABALv4i32_v2i64:
7087 case AArch64::UABALv8i16_v4i32:
7088 case AArch64::UABALv8i8_v8i16:
7089 case AArch64::UABAv16i8:
7090 case AArch64::UABAv2i32:
7091 case AArch64::UABAv4i16:
7092 case AArch64::UABAv4i32:
7093 case AArch64::UABAv8i16:
7094 case AArch64::UABAv8i8:
7095 case AArch64::SABALv16i8_v8i16:
7096 case AArch64::SABALv2i32_v2i64:
7097 case AArch64::SABALv4i16_v4i32:
7098 case AArch64::SABALv4i32_v2i64:
7099 case AArch64::SABALv8i16_v4i32:
7100 case AArch64::SABALv8i8_v8i16:
7101 case AArch64::SABAv16i8:
7102 case AArch64::SABAv2i32:
7103 case AArch64::SABAv4i16:
7104 case AArch64::SABAv4i32:
7105 case AArch64::SABAv8i16:
7106 case AArch64::SABAv8i8:
7113unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7114 unsigned AccumulationOpcode)
const {
7115 switch (AccumulationOpcode) {
7118 case AArch64::UABALB_ZZZ_D:
7119 return AArch64::UABDLB_ZZZ_D;
7120 case AArch64::UABALB_ZZZ_H:
7121 return AArch64::UABDLB_ZZZ_H;
7122 case AArch64::UABALB_ZZZ_S:
7123 return AArch64::UABDLB_ZZZ_S;
7124 case AArch64::UABALT_ZZZ_D:
7125 return AArch64::UABDLT_ZZZ_D;
7126 case AArch64::UABALT_ZZZ_H:
7127 return AArch64::UABDLT_ZZZ_H;
7128 case AArch64::UABALT_ZZZ_S:
7129 return AArch64::UABDLT_ZZZ_S;
7130 case AArch64::UABALv16i8_v8i16:
7131 return AArch64::UABDLv16i8_v8i16;
7132 case AArch64::UABALv2i32_v2i64:
7133 return AArch64::UABDLv2i32_v2i64;
7134 case AArch64::UABALv4i16_v4i32:
7135 return AArch64::UABDLv4i16_v4i32;
7136 case AArch64::UABALv4i32_v2i64:
7137 return AArch64::UABDLv4i32_v2i64;
7138 case AArch64::UABALv8i16_v4i32:
7139 return AArch64::UABDLv8i16_v4i32;
7140 case AArch64::UABALv8i8_v8i16:
7141 return AArch64::UABDLv8i8_v8i16;
7142 case AArch64::UABAv16i8:
7143 return AArch64::UABDv16i8;
7144 case AArch64::UABAv2i32:
7145 return AArch64::UABDv2i32;
7146 case AArch64::UABAv4i16:
7147 return AArch64::UABDv4i16;
7148 case AArch64::UABAv4i32:
7149 return AArch64::UABDv4i32;
7150 case AArch64::UABAv8i16:
7151 return AArch64::UABDv8i16;
7152 case AArch64::UABAv8i8:
7153 return AArch64::UABDv8i8;
7154 case AArch64::SABALB_ZZZ_D:
7155 return AArch64::SABDLB_ZZZ_D;
7156 case AArch64::SABALB_ZZZ_S:
7157 return AArch64::SABDLB_ZZZ_S;
7158 case AArch64::SABALB_ZZZ_H:
7159 return AArch64::SABDLB_ZZZ_H;
7160 case AArch64::SABALT_ZZZ_D:
7161 return AArch64::SABDLT_ZZZ_D;
7162 case AArch64::SABALT_ZZZ_S:
7163 return AArch64::SABDLT_ZZZ_S;
7164 case AArch64::SABALT_ZZZ_H:
7165 return AArch64::SABDLT_ZZZ_H;
7166 case AArch64::SABALv16i8_v8i16:
7167 return AArch64::SABDLv16i8_v8i16;
7168 case AArch64::SABALv2i32_v2i64:
7169 return AArch64::SABDLv2i32_v2i64;
7170 case AArch64::SABALv4i16_v4i32:
7171 return AArch64::SABDLv4i16_v4i32;
7172 case AArch64::SABALv4i32_v2i64:
7173 return AArch64::SABDLv4i32_v2i64;
7174 case AArch64::SABALv8i16_v4i32:
7175 return AArch64::SABDLv8i16_v4i32;
7176 case AArch64::SABALv8i8_v8i16:
7177 return AArch64::SABDLv8i8_v8i16;
7178 case AArch64::SABAv16i8:
7179 return AArch64::SABDv16i8;
7180 case AArch64::SABAv2i32:
7181 return AArch64::SABAv2i32;
7182 case AArch64::SABAv4i16:
7183 return AArch64::SABDv4i16;
7184 case AArch64::SABAv4i32:
7185 return AArch64::SABDv4i32;
7186 case AArch64::SABAv8i16:
7187 return AArch64::SABDv8i16;
7188 case AArch64::SABAv8i8:
7189 return AArch64::SABDv8i8;
7205 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
7217 assert(
false &&
"Unsupported FP instruction in combiner\n");
7219 case AArch64::FADDHrr:
7221 "FADDHrr does not have register operands");
7223 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
7224 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
7226 case AArch64::FADDSrr:
7228 "FADDSrr does not have register operands");
7230 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
7231 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
7233 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
7234 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
7236 case AArch64::FADDDrr:
7237 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
7238 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
7240 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
7241 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
7243 case AArch64::FADDv4f16:
7244 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
7245 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
7247 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
7248 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
7250 case AArch64::FADDv8f16:
7251 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
7252 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
7254 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
7255 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
7257 case AArch64::FADDv2f32:
7258 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
7259 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
7261 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
7262 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
7264 case AArch64::FADDv2f64:
7265 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
7266 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
7268 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
7269 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
7271 case AArch64::FADDv4f32:
7272 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
7273 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
7275 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
7276 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
7278 case AArch64::FSUBHrr:
7279 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
7280 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
7281 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
7283 case AArch64::FSUBSrr:
7284 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
7286 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
7287 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
7289 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
7291 case AArch64::FSUBDrr:
7292 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
7294 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
7295 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
7297 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
7299 case AArch64::FSUBv4f16:
7300 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
7301 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
7303 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
7304 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
7306 case AArch64::FSUBv8f16:
7307 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
7308 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
7310 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
7311 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
7313 case AArch64::FSUBv2f32:
7314 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
7315 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
7317 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
7318 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
7320 case AArch64::FSUBv2f64:
7321 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
7322 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
7324 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
7325 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
7327 case AArch64::FSUBv4f32:
7328 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
7329 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
7331 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
7332 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
7343 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
7350 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
7351 MI->getOperand(1).getReg().isVirtual())
7352 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
7353 if (
MI &&
MI->getOpcode() == Opcode) {
7365 case AArch64::FMULv2f32:
7366 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
7367 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
7369 case AArch64::FMULv2f64:
7370 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
7371 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
7373 case AArch64::FMULv4f16:
7374 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
7375 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
7377 case AArch64::FMULv4f32:
7378 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
7379 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
7381 case AArch64::FMULv8f16:
7382 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
7383 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
7396 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
7399 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
7400 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
7414 case AArch64::FNEGDr:
7416 case AArch64::FNEGSr:
7548 case AArch64::SUBWrr:
7549 case AArch64::SUBSWrr:
7550 case AArch64::SUBXrr:
7551 case AArch64::SUBSXrr:
7596 unsigned LoadLaneOpCode,
unsigned NumLanes) {
7619 while (!RemainingLanes.
empty() && CurrInstr &&
7620 CurrInstr->getOpcode() == LoadLaneOpCode &&
7621 MRI.hasOneNonDBGUse(CurrInstr->getOperand(0).getReg()) &&
7622 CurrInstr->getNumOperands() == 4) {
7623 RemainingLanes.
erase(CurrInstr->getOperand(2).getImm());
7625 CurrInstr =
MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
7629 if (!RemainingLanes.
empty())
7633 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
7637 auto Lane0LoadReg = CurrInstr->getOperand(2).getReg();
7638 unsigned SingleLaneSizeInBits = 128 / NumLanes;
7639 if (
TRI->getRegSizeInBits(Lane0LoadReg,
MRI) != SingleLaneSizeInBits)
7643 if (!
MRI.hasOneNonDBGUse(Lane0LoadReg))
7646 LoadInstrs.
push_back(
MRI.getUniqueVRegDef(Lane0LoadReg));
7655 RemainingLoadInstrs.
insert(LoadInstrs.
begin(), LoadInstrs.
end());
7658 for (; MBBItr !=
MBB->begin() && RemainingSteps > 0 &&
7659 !RemainingLoadInstrs.
empty();
7660 --MBBItr, --RemainingSteps) {
7664 RemainingLoadInstrs.
erase(&CurrInstr);
7674 if (RemainingSteps == 0 && !RemainingLoadInstrs.
empty())
7700 case AArch64::LD1i32:
7702 case AArch64::LD1i16:
7704 case AArch64::LD1i8:
7720 unsigned Pattern,
unsigned NumLanes) {
7728 for (
unsigned i = 0; i < NumLanes - 1; ++i) {
7736 return A->getOperand(2).getImm() >
B->getOperand(2).getImm();
7741 MRI.getUniqueVRegDef(SubregToReg->getOperand(2).getReg()));
7742 auto LoadToLaneInstrsAscending =
llvm::reverse(LoadToLaneInstrs);
7748 auto CreateLD1Instruction = [&](
MachineInstr *OriginalInstr,
7749 Register SrcRegister,
unsigned Lane,
7751 bool OffsetRegisterKillState) {
7752 auto NewRegister =
MRI.createVirtualRegister(FPR128RegClass);
7759 InstrIdxForVirtReg.
insert(std::make_pair(NewRegister, InsInstrs.
size()));
7760 InsInstrs.
push_back(LoadIndexIntoRegister);
7766 auto CreateLDRInstruction = [&](
unsigned NumLanes,
Register DestReg,
7772 Opcode = AArch64::LDRSui;
7775 Opcode = AArch64::LDRHui;
7778 Opcode = AArch64::LDRBui;
7782 "Got unsupported number of lanes in machine-combiner gather pattern");
7791 auto LanesToLoadToReg0 =
7793 LoadToLaneInstrsAscending.begin() + NumLanes / 2);
7794 Register PrevReg = SubregToReg->getOperand(0).getReg();
7796 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
7797 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
7798 OffsetRegOperand.
getReg(),
7799 OffsetRegOperand.
isKill());
7806 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
7808 *std::next(LoadToLaneInstrsAscending.begin(), NumLanes / 2);
7809 Register DestRegForMiddleIndex =
MRI.createVirtualRegister(
7815 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
7816 OriginalSplitToLoadOffsetOperand.
getReg(),
7817 OriginalSplitToLoadOffsetOperand.
isKill());
7819 InstrIdxForVirtReg.
insert(
7820 std::make_pair(DestRegForMiddleIndex, InsInstrs.
size()));
7821 InsInstrs.
push_back(MiddleIndexLoadInstr);
7825 Register DestRegForSubregToReg =
MRI.createVirtualRegister(FPR128RegClass);
7826 unsigned SubregType;
7829 SubregType = AArch64::ssub;
7832 SubregType = AArch64::hsub;
7835 SubregType = AArch64::bsub;
7839 "Got invalid NumLanes for machine-combiner gather pattern");
7842 auto SubRegToRegInstr =
7844 DestRegForSubregToReg)
7848 InstrIdxForVirtReg.
insert(
7849 std::make_pair(DestRegForSubregToReg, InsInstrs.
size()));
7853 auto LanesToLoadToReg1 =
7855 LoadToLaneInstrsAscending.end());
7856 PrevReg = SubRegToRegInstr->getOperand(0).getReg();
7858 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
7859 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
7860 OffsetRegOperand.
getReg(),
7861 OffsetRegOperand.
isKill());
7864 if (Index == NumLanes / 2 - 2) {
7899bool AArch64InstrInfo::getMachineCombinerPatterns(
7901 bool DoRegPressureReduce)
const {
7922 DoRegPressureReduce);
7951 const Register *ReplacedAddend =
nullptr) {
7952 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7954 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
7957 Register SrcReg0 = MUL->getOperand(1).getReg();
7958 bool Src0IsKill = MUL->getOperand(1).isKill();
7959 Register SrcReg1 = MUL->getOperand(2).getReg();
7960 bool Src1IsKill = MUL->getOperand(2).isKill();
7964 if (ReplacedAddend) {
7966 SrcReg2 = *ReplacedAddend;
7974 MRI.constrainRegClass(ResultReg, RC);
7976 MRI.constrainRegClass(SrcReg0, RC);
7978 MRI.constrainRegClass(SrcReg1, RC);
7980 MRI.constrainRegClass(SrcReg2, RC);
7993 .
addImm(MUL->getOperand(3).getImm());
8000 assert(
false &&
"Invalid FMA instruction kind \n");
8014 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8015 Opc = AArch64::FNMADDSrrr;
8016 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8017 Opc = AArch64::FNMADDDrrr;
8029 MRI.constrainRegClass(ResultReg, RC);
8031 MRI.constrainRegClass(SrcReg0, RC);
8033 MRI.constrainRegClass(SrcReg1, RC);
8035 MRI.constrainRegClass(SrcReg2, RC);
8051 unsigned IdxDupOp,
unsigned MulOpc,
8053 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8054 "Invalid index of FMUL operand");
8062 if (Dup->
getOpcode() == TargetOpcode::COPY)
8066 MRI.clearKillFlags(DupSrcReg);
8067 MRI.constrainRegClass(DupSrcReg, RC);
8071 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8112 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8127 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8154 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8182 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
8184 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8188 Register SrcReg0 = MUL->getOperand(1).getReg();
8189 bool Src0IsKill = MUL->getOperand(1).isKill();
8190 Register SrcReg1 = MUL->getOperand(2).getReg();
8191 bool Src1IsKill = MUL->getOperand(2).isKill();
8194 MRI.constrainRegClass(ResultReg, RC);
8196 MRI.constrainRegClass(SrcReg0, RC);
8198 MRI.constrainRegClass(SrcReg1, RC);
8200 MRI.constrainRegClass(VR, RC);
8221 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
8222 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
8236 if (Opcode == AArch64::SUBSWrr)
8237 Opcode = AArch64::SUBWrr;
8238 else if (Opcode == AArch64::SUBSXrr)
8239 Opcode = AArch64::SUBXrr;
8241 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
8242 "Unexpected instruction opcode.");
8259 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8266unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
8267 unsigned int AccumulatorOpCode)
const {
8268 switch (AccumulatorOpCode) {
8269 case AArch64::UABALB_ZZZ_D:
8270 case AArch64::SABALB_ZZZ_D:
8271 case AArch64::UABALT_ZZZ_D:
8272 case AArch64::SABALT_ZZZ_D:
8273 return AArch64::ADD_ZZZ_D;
8274 case AArch64::UABALB_ZZZ_H:
8275 case AArch64::SABALB_ZZZ_H:
8276 case AArch64::UABALT_ZZZ_H:
8277 case AArch64::SABALT_ZZZ_H:
8278 return AArch64::ADD_ZZZ_H;
8279 case AArch64::UABALB_ZZZ_S:
8280 case AArch64::SABALB_ZZZ_S:
8281 case AArch64::UABALT_ZZZ_S:
8282 case AArch64::SABALT_ZZZ_S:
8283 return AArch64::ADD_ZZZ_S;
8284 case AArch64::UABALv16i8_v8i16:
8285 case AArch64::SABALv8i8_v8i16:
8286 case AArch64::SABAv8i16:
8287 case AArch64::UABAv8i16:
8288 return AArch64::ADDv8i16;
8289 case AArch64::SABALv2i32_v2i64:
8290 case AArch64::UABALv2i32_v2i64:
8291 case AArch64::SABALv4i32_v2i64:
8292 return AArch64::ADDv2i64;
8293 case AArch64::UABALv4i16_v4i32:
8294 case AArch64::SABALv4i16_v4i32:
8295 case AArch64::SABALv8i16_v4i32:
8296 case AArch64::SABAv4i32:
8297 case AArch64::UABAv4i32:
8298 return AArch64::ADDv4i32;
8299 case AArch64::UABALv4i32_v2i64:
8300 return AArch64::ADDv2i64;
8301 case AArch64::UABALv8i16_v4i32:
8302 return AArch64::ADDv4i32;
8303 case AArch64::UABALv8i8_v8i16:
8304 case AArch64::SABALv16i8_v8i16:
8305 return AArch64::ADDv8i16;
8306 case AArch64::UABAv16i8:
8307 case AArch64::SABAv16i8:
8308 return AArch64::ADDv16i8;
8309 case AArch64::UABAv4i16:
8310 case AArch64::SABAv4i16:
8311 return AArch64::ADDv4i16;
8312 case AArch64::UABAv2i32:
8313 case AArch64::SABAv2i32:
8314 return AArch64::ADDv2i32;
8315 case AArch64::UABAv8i8:
8316 case AArch64::SABAv8i8:
8317 return AArch64::ADDv8i8;
8326void AArch64InstrInfo::genAlternativeCodeSequence(
8336 MachineInstr *
MUL =
nullptr;
8337 const TargetRegisterClass *RC;
8343 DelInstrs, InstrIdxForVirtReg);
8349 InstrIdxForVirtReg);
8355 InstrIdxForVirtReg);
8364 Opc = AArch64::MADDWrrr;
8365 RC = &AArch64::GPR32RegClass;
8367 Opc = AArch64::MADDXrrr;
8368 RC = &AArch64::GPR64RegClass;
8379 Opc = AArch64::MADDWrrr;
8380 RC = &AArch64::GPR32RegClass;
8382 Opc = AArch64::MADDXrrr;
8383 RC = &AArch64::GPR64RegClass;
8396 const TargetRegisterClass *RC;
8397 unsigned BitSize, MovImm;
8400 MovImm = AArch64::MOVi32imm;
8401 RC = &AArch64::GPR32spRegClass;
8403 Opc = AArch64::MADDWrrr;
8404 RC = &AArch64::GPR32RegClass;
8406 MovImm = AArch64::MOVi64imm;
8407 RC = &AArch64::GPR64spRegClass;
8409 Opc = AArch64::MADDXrrr;
8410 RC = &AArch64::GPR64RegClass;
8421 uint64_t UImm =
SignExtend64(IsSub ? -Imm : Imm, BitSize);
8425 if (Insn.
size() != 1)
8427 MachineInstrBuilder MIB1 =
8428 BuildMI(MF, MIMetadata(Root),
TII->get(MovImm), NewVR)
8429 .
addImm(IsSub ? -Imm : Imm);
8431 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8442 const TargetRegisterClass *SubRC;
8443 unsigned SubOpc, ZeroReg;
8445 SubOpc = AArch64::SUBWrr;
8446 SubRC = &AArch64::GPR32spRegClass;
8447 ZeroReg = AArch64::WZR;
8448 Opc = AArch64::MADDWrrr;
8449 RC = &AArch64::GPR32RegClass;
8451 SubOpc = AArch64::SUBXrr;
8452 SubRC = &AArch64::GPR64spRegClass;
8453 ZeroReg = AArch64::XZR;
8454 Opc = AArch64::MADDXrrr;
8455 RC = &AArch64::GPR64RegClass;
8457 Register NewVR =
MRI.createVirtualRegister(SubRC);
8459 MachineInstrBuilder MIB1 =
8460 BuildMI(MF, MIMetadata(Root),
TII->get(SubOpc), NewVR)
8464 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8475 Opc = AArch64::MSUBWrrr;
8476 RC = &AArch64::GPR32RegClass;
8478 Opc = AArch64::MSUBXrrr;
8479 RC = &AArch64::GPR64RegClass;
8484 Opc = AArch64::MLAv8i8;
8485 RC = &AArch64::FPR64RegClass;
8489 Opc = AArch64::MLAv8i8;
8490 RC = &AArch64::FPR64RegClass;
8494 Opc = AArch64::MLAv16i8;
8495 RC = &AArch64::FPR128RegClass;
8499 Opc = AArch64::MLAv16i8;
8500 RC = &AArch64::FPR128RegClass;
8504 Opc = AArch64::MLAv4i16;
8505 RC = &AArch64::FPR64RegClass;
8509 Opc = AArch64::MLAv4i16;
8510 RC = &AArch64::FPR64RegClass;
8514 Opc = AArch64::MLAv8i16;
8515 RC = &AArch64::FPR128RegClass;
8519 Opc = AArch64::MLAv8i16;
8520 RC = &AArch64::FPR128RegClass;
8524 Opc = AArch64::MLAv2i32;
8525 RC = &AArch64::FPR64RegClass;
8529 Opc = AArch64::MLAv2i32;
8530 RC = &AArch64::FPR64RegClass;
8534 Opc = AArch64::MLAv4i32;
8535 RC = &AArch64::FPR128RegClass;
8539 Opc = AArch64::MLAv4i32;
8540 RC = &AArch64::FPR128RegClass;
8545 Opc = AArch64::MLAv8i8;
8546 RC = &AArch64::FPR64RegClass;
8548 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i8,
8552 Opc = AArch64::MLSv8i8;
8553 RC = &AArch64::FPR64RegClass;
8557 Opc = AArch64::MLAv16i8;
8558 RC = &AArch64::FPR128RegClass;
8560 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv16i8,
8564 Opc = AArch64::MLSv16i8;
8565 RC = &AArch64::FPR128RegClass;
8569 Opc = AArch64::MLAv4i16;
8570 RC = &AArch64::FPR64RegClass;
8572 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i16,
8576 Opc = AArch64::MLSv4i16;
8577 RC = &AArch64::FPR64RegClass;
8581 Opc = AArch64::MLAv8i16;
8582 RC = &AArch64::FPR128RegClass;
8584 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i16,
8588 Opc = AArch64::MLSv8i16;
8589 RC = &AArch64::FPR128RegClass;
8593 Opc = AArch64::MLAv2i32;
8594 RC = &AArch64::FPR64RegClass;
8596 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv2i32,
8600 Opc = AArch64::MLSv2i32;
8601 RC = &AArch64::FPR64RegClass;
8605 Opc = AArch64::MLAv4i32;
8606 RC = &AArch64::FPR128RegClass;
8608 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i32,
8612 Opc = AArch64::MLSv4i32;
8613 RC = &AArch64::FPR128RegClass;
8618 Opc = AArch64::MLAv4i16_indexed;
8619 RC = &AArch64::FPR64RegClass;
8623 Opc = AArch64::MLAv4i16_indexed;
8624 RC = &AArch64::FPR64RegClass;
8628 Opc = AArch64::MLAv8i16_indexed;
8629 RC = &AArch64::FPR128RegClass;
8633 Opc = AArch64::MLAv8i16_indexed;
8634 RC = &AArch64::FPR128RegClass;
8638 Opc = AArch64::MLAv2i32_indexed;
8639 RC = &AArch64::FPR64RegClass;
8643 Opc = AArch64::MLAv2i32_indexed;
8644 RC = &AArch64::FPR64RegClass;
8648 Opc = AArch64::MLAv4i32_indexed;
8649 RC = &AArch64::FPR128RegClass;
8653 Opc = AArch64::MLAv4i32_indexed;
8654 RC = &AArch64::FPR128RegClass;
8659 Opc = AArch64::MLAv4i16_indexed;
8660 RC = &AArch64::FPR64RegClass;
8662 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i16,
8666 Opc = AArch64::MLSv4i16_indexed;
8667 RC = &AArch64::FPR64RegClass;
8671 Opc = AArch64::MLAv8i16_indexed;
8672 RC = &AArch64::FPR128RegClass;
8674 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i16,
8678 Opc = AArch64::MLSv8i16_indexed;
8679 RC = &AArch64::FPR128RegClass;
8683 Opc = AArch64::MLAv2i32_indexed;
8684 RC = &AArch64::FPR64RegClass;
8686 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv2i32,
8690 Opc = AArch64::MLSv2i32_indexed;
8691 RC = &AArch64::FPR64RegClass;
8695 Opc = AArch64::MLAv4i32_indexed;
8696 RC = &AArch64::FPR128RegClass;
8698 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i32,
8702 Opc = AArch64::MLSv4i32_indexed;
8703 RC = &AArch64::FPR128RegClass;
8709 Opc = AArch64::FMADDHrrr;
8710 RC = &AArch64::FPR16RegClass;
8714 Opc = AArch64::FMADDSrrr;
8715 RC = &AArch64::FPR32RegClass;
8719 Opc = AArch64::FMADDDrrr;
8720 RC = &AArch64::FPR64RegClass;
8725 Opc = AArch64::FMADDHrrr;
8726 RC = &AArch64::FPR16RegClass;
8730 Opc = AArch64::FMADDSrrr;
8731 RC = &AArch64::FPR32RegClass;
8735 Opc = AArch64::FMADDDrrr;
8736 RC = &AArch64::FPR64RegClass;
8741 Opc = AArch64::FMLAv1i32_indexed;
8742 RC = &AArch64::FPR32RegClass;
8747 Opc = AArch64::FMLAv1i32_indexed;
8748 RC = &AArch64::FPR32RegClass;
8754 Opc = AArch64::FMLAv1i64_indexed;
8755 RC = &AArch64::FPR64RegClass;
8760 Opc = AArch64::FMLAv1i64_indexed;
8761 RC = &AArch64::FPR64RegClass;
8767 RC = &AArch64::FPR64RegClass;
8768 Opc = AArch64::FMLAv4i16_indexed;
8773 RC = &AArch64::FPR64RegClass;
8774 Opc = AArch64::FMLAv4f16;
8779 RC = &AArch64::FPR64RegClass;
8780 Opc = AArch64::FMLAv4i16_indexed;
8785 RC = &AArch64::FPR64RegClass;
8786 Opc = AArch64::FMLAv4f16;
8793 RC = &AArch64::FPR64RegClass;
8795 Opc = AArch64::FMLAv2i32_indexed;
8799 Opc = AArch64::FMLAv2f32;
8806 RC = &AArch64::FPR64RegClass;
8808 Opc = AArch64::FMLAv2i32_indexed;
8812 Opc = AArch64::FMLAv2f32;
8819 RC = &AArch64::FPR128RegClass;
8820 Opc = AArch64::FMLAv8i16_indexed;
8825 RC = &AArch64::FPR128RegClass;
8826 Opc = AArch64::FMLAv8f16;
8831 RC = &AArch64::FPR128RegClass;
8832 Opc = AArch64::FMLAv8i16_indexed;
8837 RC = &AArch64::FPR128RegClass;
8838 Opc = AArch64::FMLAv8f16;
8845 RC = &AArch64::FPR128RegClass;
8847 Opc = AArch64::FMLAv2i64_indexed;
8851 Opc = AArch64::FMLAv2f64;
8858 RC = &AArch64::FPR128RegClass;
8860 Opc = AArch64::FMLAv2i64_indexed;
8864 Opc = AArch64::FMLAv2f64;
8872 RC = &AArch64::FPR128RegClass;
8874 Opc = AArch64::FMLAv4i32_indexed;
8878 Opc = AArch64::FMLAv4f32;
8886 RC = &AArch64::FPR128RegClass;
8888 Opc = AArch64::FMLAv4i32_indexed;
8892 Opc = AArch64::FMLAv4f32;
8899 Opc = AArch64::FNMSUBHrrr;
8900 RC = &AArch64::FPR16RegClass;
8904 Opc = AArch64::FNMSUBSrrr;
8905 RC = &AArch64::FPR32RegClass;
8909 Opc = AArch64::FNMSUBDrrr;
8910 RC = &AArch64::FPR64RegClass;
8915 Opc = AArch64::FNMADDHrrr;
8916 RC = &AArch64::FPR16RegClass;
8920 Opc = AArch64::FNMADDSrrr;
8921 RC = &AArch64::FPR32RegClass;
8925 Opc = AArch64::FNMADDDrrr;
8926 RC = &AArch64::FPR64RegClass;
8931 Opc = AArch64::FMSUBHrrr;
8932 RC = &AArch64::FPR16RegClass;
8936 Opc = AArch64::FMSUBSrrr;
8937 RC = &AArch64::FPR32RegClass;
8941 Opc = AArch64::FMSUBDrrr;
8942 RC = &AArch64::FPR64RegClass;
8947 Opc = AArch64::FMLSv1i32_indexed;
8948 RC = &AArch64::FPR32RegClass;
8954 Opc = AArch64::FMLSv1i64_indexed;
8955 RC = &AArch64::FPR64RegClass;
8962 RC = &AArch64::FPR64RegClass;
8964 MachineInstrBuilder MIB1 =
8965 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv4f16), NewVR)
8968 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8970 Opc = AArch64::FMLAv4f16;
8974 Opc = AArch64::FMLAv4i16_indexed;
8981 RC = &AArch64::FPR64RegClass;
8982 Opc = AArch64::FMLSv4f16;
8987 RC = &AArch64::FPR64RegClass;
8988 Opc = AArch64::FMLSv4i16_indexed;
8995 RC = &AArch64::FPR64RegClass;
8997 Opc = AArch64::FMLSv2i32_indexed;
9001 Opc = AArch64::FMLSv2f32;
9009 RC = &AArch64::FPR128RegClass;
9011 MachineInstrBuilder MIB1 =
9012 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv8f16), NewVR)
9015 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9017 Opc = AArch64::FMLAv8f16;
9021 Opc = AArch64::FMLAv8i16_indexed;
9028 RC = &AArch64::FPR128RegClass;
9029 Opc = AArch64::FMLSv8f16;
9034 RC = &AArch64::FPR128RegClass;
9035 Opc = AArch64::FMLSv8i16_indexed;
9042 RC = &AArch64::FPR128RegClass;
9044 Opc = AArch64::FMLSv2i64_indexed;
9048 Opc = AArch64::FMLSv2f64;
9056 RC = &AArch64::FPR128RegClass;
9058 Opc = AArch64::FMLSv4i32_indexed;
9062 Opc = AArch64::FMLSv4f32;
9069 RC = &AArch64::FPR64RegClass;
9071 MachineInstrBuilder MIB1 =
9072 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv2f32), NewVR)
9075 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9077 Opc = AArch64::FMLAv2i32_indexed;
9081 Opc = AArch64::FMLAv2f32;
9089 RC = &AArch64::FPR128RegClass;
9091 MachineInstrBuilder MIB1 =
9092 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv4f32), NewVR)
9095 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9097 Opc = AArch64::FMLAv4i32_indexed;
9101 Opc = AArch64::FMLAv4f32;
9109 RC = &AArch64::FPR128RegClass;
9111 MachineInstrBuilder MIB1 =
9112 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv2f64), NewVR)
9115 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9117 Opc = AArch64::FMLAv2i64_indexed;
9121 Opc = AArch64::FMLAv2f64;
9133 &AArch64::FPR128RegClass,
MRI);
9142 &AArch64::FPR128RegClass,
MRI);
9151 &AArch64::FPR128_loRegClass,
MRI);
9160 &AArch64::FPR128RegClass,
MRI);
9169 &AArch64::FPR128_loRegClass,
MRI);
9203 for (
auto *
MI : InsInstrs)
9204 MI->setFlags(Flags);
9245 bool IsNegativeBranch =
false;
9246 bool IsTestAndBranch =
false;
9247 unsigned TargetBBInMI = 0;
9248 switch (
MI.getOpcode()) {
9252 case AArch64::CBWPri:
9253 case AArch64::CBXPri:
9254 case AArch64::CBWPrr:
9255 case AArch64::CBXPrr:
9261 case AArch64::CBNZW:
9262 case AArch64::CBNZX:
9264 IsNegativeBranch =
true;
9269 IsTestAndBranch =
true;
9271 case AArch64::TBNZW:
9272 case AArch64::TBNZX:
9274 IsNegativeBranch =
true;
9275 IsTestAndBranch =
true;
9281 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
9285 assert(
MI.getParent() &&
"Incomplete machine instruction\n");
9296 while (
DefMI->isCopy()) {
9298 if (!
MRI->hasOneNonDBGUse(CopyVReg))
9300 if (!
MRI->hasOneDef(CopyVReg))
9305 switch (
DefMI->getOpcode()) {
9309 case AArch64::ANDWri:
9310 case AArch64::ANDXri: {
9311 if (IsTestAndBranch)
9315 if (!
MRI->hasOneNonDBGUse(VReg))
9318 bool Is32Bit = (
DefMI->getOpcode() == AArch64::ANDWri);
9320 DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
9329 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
9335 unsigned Opc = (Imm < 32)
9336 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
9337 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
9350 if (!Is32Bit && Imm < 32)
9352 MI.eraseFromParent();
9356 case AArch64::CSINCWr:
9357 case AArch64::CSINCXr: {
9358 if (!(
DefMI->getOperand(1).getReg() == AArch64::WZR &&
9359 DefMI->getOperand(2).getReg() == AArch64::WZR) &&
9360 !(
DefMI->getOperand(1).getReg() == AArch64::XZR &&
9361 DefMI->getOperand(2).getReg() == AArch64::XZR))
9364 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
9377 if (IsNegativeBranch)
9380 MI.eraseFromParent();
9386std::pair<unsigned, unsigned>
9387AArch64InstrInfo::decomposeMachineOperandsTargetFlags(
unsigned TF)
const {
9389 return std::make_pair(TF & Mask, TF & ~Mask);
9393AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags()
const {
9396 static const std::pair<unsigned, const char *> TargetFlags[] = {
9397 {MO_PAGE,
"aarch64-page"}, {
MO_PAGEOFF,
"aarch64-pageoff"},
9398 {
MO_G3,
"aarch64-g3"}, {
MO_G2,
"aarch64-g2"},
9399 {
MO_G1,
"aarch64-g1"}, {
MO_G0,
"aarch64-g0"},
9405AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags()
const {
9406 using namespace AArch64II;
9408 static const std::pair<unsigned, const char *> TargetFlags[] = {
9411 {
MO_NC,
"aarch64-nc"},
9412 {
MO_S,
"aarch64-s"},
9423AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags()
const {
9424 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
9526 MachineFunction *MF =
C.getMF();
9528 const AArch64RegisterInfo *ARI =
9529 static_cast<const AArch64RegisterInfo *
>(&
TRI);
9532 for (
unsigned Reg : AArch64::GPR64RegClass) {
9534 Reg != AArch64::LR &&
9535 Reg != AArch64::X16 &&
9536 Reg != AArch64::X17 &&
9537 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
9538 C.isAvailableInsideSeq(
Reg,
TRI))
9569 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
9572std::optional<std::unique_ptr<outliner::OutlinedFunction>>
9573AArch64InstrInfo::getOutliningCandidateInfo(
9575 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
9576 unsigned MinRepeats)
const {
9577 unsigned SequenceSize = 0;
9578 for (
auto &
MI : RepeatedSequenceLocs[0])
9581 unsigned NumBytesToCreateFrame = 0;
9591 if (std::adjacent_find(
9592 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
9593 [](
const outliner::Candidate &a,
const outliner::Candidate &b) {
9596 if (outliningCandidatesSigningScopeConsensus(a, b) &&
9597 outliningCandidatesSigningKeyConsensus(a, b) &&
9598 outliningCandidatesV8_3OpsConsensus(a, b)) {
9602 }) != RepeatedSequenceLocs.end()) {
9603 return std::nullopt;
9620 unsigned NumBytesToCheckLRInTCEpilogue = 0;
9621 if (RepeatedSequenceLocs[0]
9623 ->getInfo<AArch64FunctionInfo>()
9624 ->shouldSignReturnAddress(
true)) {
9626 NumBytesToCreateFrame += 8;
9629 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
9630 *RepeatedSequenceLocs[0].getMF());
9631 NumBytesToCheckLRInTCEpilogue =
9635 if (isTailCallReturnInst(RepeatedSequenceLocs[0].
back()))
9636 SequenceSize += NumBytesToCheckLRInTCEpilogue;
9644 for (
auto &
MI :
C) {
9645 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
9646 switch (
MI.getOpcode()) {
9647 case AArch64::ADDXri:
9648 case AArch64::ADDWri:
9649 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
9651 "Expected operand to be immediate");
9653 "Expected operand to be a register");
9657 if (
MI.getOperand(1).getReg() == AArch64::SP)
9658 SPValue +=
MI.getOperand(2).getImm();
9662 case AArch64::SUBXri:
9663 case AArch64::SUBWri:
9664 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
9666 "Expected operand to be immediate");
9668 "Expected operand to be a register");
9672 if (
MI.getOperand(1).getReg() == AArch64::SP)
9673 SPValue -=
MI.getOperand(2).getImm();
9690 if (RepeatedSequenceLocs.size() < MinRepeats)
9691 return std::nullopt;
9695 unsigned FlagsSetInAll = 0xF;
9699 FlagsSetInAll &=
C.Flags;
9701 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
9704 auto SetCandidateCallInfo =
9705 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
9707 C.setCallInfo(CallID, NumBytesForCall);
9711 NumBytesToCreateFrame += 4;
9719 unsigned CFICount = 0;
9720 for (
auto &
I : RepeatedSequenceLocs[0]) {
9721 if (
I.isCFIInstruction())
9731 std::vector<MCCFIInstruction> CFIInstructions =
9732 C.getMF()->getFrameInstructions();
9734 if (CFICount > 0 && CFICount != CFIInstructions.size())
9735 return std::nullopt;
9743 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
9744 !
MI.readsRegister(AArch64::SP, &
TRI))
9750 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
9755 if (
MI.mayLoadOrStore()) {
9758 bool OffsetIsScalable;
9762 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
9763 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
9767 if (OffsetIsScalable)
9775 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
9776 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
9779 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
9780 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
9795 bool AllStackInstrsSafe =
9800 if (RepeatedSequenceLocs[0].
back().isTerminator()) {
9802 NumBytesToCreateFrame = 0;
9803 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
9807 else if (LastInstrOpcode == AArch64::BL ||
9808 ((LastInstrOpcode == AArch64::BLR ||
9809 LastInstrOpcode == AArch64::BLRNoIP) &&
9813 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
9821 unsigned NumBytesNoStackCalls = 0;
9822 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
9828 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
9837 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
9840 if (LRAvailable && !IsNoReturn) {
9841 NumBytesNoStackCalls += 4;
9843 CandidatesWithoutStackFixups.push_back(
C);
9848 else if (findRegisterToSaveLRTo(
C)) {
9849 NumBytesNoStackCalls += 12;
9851 CandidatesWithoutStackFixups.push_back(
C);
9856 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
9857 NumBytesNoStackCalls += 12;
9859 CandidatesWithoutStackFixups.push_back(
C);
9865 NumBytesNoStackCalls += SequenceSize;
9872 if (!AllStackInstrsSafe ||
9873 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
9874 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
9876 if (RepeatedSequenceLocs.size() < MinRepeats)
9877 return std::nullopt;
9930 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
9931 !findRegisterToSaveLRTo(
C));
9937 if (RepeatedSequenceLocs.size() < MinRepeats)
9938 return std::nullopt;
9947 bool ModStackToSaveLR =
false;
9950 ModStackToSaveLR =
true;
9959 ModStackToSaveLR =
true;
9961 if (ModStackToSaveLR) {
9963 if (!AllStackInstrsSafe)
9964 return std::nullopt;
9967 NumBytesToCreateFrame += 8;
9974 return std::nullopt;
9976 return std::make_unique<outliner::OutlinedFunction>(
9977 RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID);
9980void AArch64InstrInfo::mergeOutliningCandidateAttributes(
9981 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
9985 const auto &CFn = Candidates.front().getMF()->getFunction();
9987 if (CFn.hasFnAttribute(
"ptrauth-returns"))
9988 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-returns"));
9989 if (CFn.hasFnAttribute(
"ptrauth-auth-traps"))
9990 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-auth-traps"));
9993 if (CFn.hasFnAttribute(
"sign-return-address"))
9994 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
9995 if (CFn.hasFnAttribute(
"sign-return-address-key"))
9996 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
9998 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
10001bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10006 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
10013 if (
F.hasSection())
10019 AArch64FunctionInfo *AFI = MF.
getInfo<AArch64FunctionInfo>();
10020 if (!AFI || AFI->
hasRedZone().value_or(
true))
10040 unsigned &Flags)
const {
10042 "Must track liveness!");
10044 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10059 auto AreAllUnsafeRegsDead = [&LRU]() {
10060 return LRU.available(AArch64::W16) && LRU.available(AArch64::W17) &&
10061 LRU.available(AArch64::NZCV);
10076 bool LRAvailableEverywhere =
true;
10078 LRU.addLiveOuts(
MBB);
10080 auto UpdateWholeMBBFlags = [&
Flags](
const MachineInstr &
MI) {
10081 if (
MI.isCall() && !
MI.isTerminator())
10087 auto CreateNewRangeStartingAt =
10088 [&RangeBegin, &RangeEnd,
10090 RangeBegin = NewBegin;
10091 RangeEnd = std::next(RangeBegin);
10094 auto SaveRangeIfNonEmpty = [&RangeLen, &
Ranges, &RangeBegin, &RangeEnd]() {
10100 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
10102 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
10104 Ranges.emplace_back(RangeBegin, RangeEnd);
10112 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
10113 LRU.stepBackward(*FirstPossibleEndPt);
10116 UpdateWholeMBBFlags(*FirstPossibleEndPt);
10117 if (AreAllUnsafeRegsDead())
10124 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
10129 LRU.stepBackward(
MI);
10130 UpdateWholeMBBFlags(
MI);
10131 if (!AreAllUnsafeRegsDead()) {
10132 SaveRangeIfNonEmpty();
10133 CreateNewRangeStartingAt(
MI.getIterator());
10136 LRAvailableEverywhere &= LRU.available(AArch64::LR);
10137 RangeBegin =
MI.getIterator();
10142 if (AreAllUnsafeRegsDead())
10143 SaveRangeIfNonEmpty();
10151 if (!LRAvailableEverywhere)
10159 unsigned Flags)
const {
10160 MachineInstr &
MI = *MIT;
10164 switch (
MI.getOpcode()) {
10165 case AArch64::PACM:
10166 case AArch64::PACIASP:
10167 case AArch64::PACIBSP:
10168 case AArch64::PACIASPPC:
10169 case AArch64::PACIBSPPC:
10170 case AArch64::AUTIASP:
10171 case AArch64::AUTIBSP:
10172 case AArch64::AUTIASPPCi:
10173 case AArch64::AUTIASPPCr:
10174 case AArch64::AUTIBSPPCi:
10175 case AArch64::AUTIBSPPCr:
10176 case AArch64::RETAA:
10177 case AArch64::RETAB:
10178 case AArch64::RETAASPPCi:
10179 case AArch64::RETAASPPCr:
10180 case AArch64::RETABSPPCi:
10181 case AArch64::RETABSPPCr:
10182 case AArch64::EMITBKEY:
10183 case AArch64::PAUTH_PROLOGUE:
10184 case AArch64::PAUTH_EPILOGUE:
10194 if (
MI.isCFIInstruction())
10198 if (
MI.isTerminator())
10204 for (
const MachineOperand &MOP :
MI.operands()) {
10207 assert(!MOP.isCFIIndex());
10210 if (MOP.isReg() && !MOP.isImplicit() &&
10211 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
10218 if (
MI.getOpcode() == AArch64::ADRP)
10238 for (
const MachineOperand &MOP :
MI.operands()) {
10239 if (MOP.isGlobal()) {
10247 if (Callee &&
Callee->getName() ==
"\01_mcount")
10255 if (
MI.getOpcode() == AArch64::BLR ||
10256 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
10260 return UnknownCallOutlineType;
10268 return UnknownCallOutlineType;
10276 return UnknownCallOutlineType;
10297 for (MachineInstr &
MI :
MBB) {
10298 const MachineOperand *
Base;
10299 TypeSize Width(0,
false);
10301 bool OffsetIsScalable;
10304 if (!
MI.mayLoadOrStore() ||
10307 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
10311 TypeSize Scale(0U,
false);
10312 int64_t Dummy1, Dummy2;
10315 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
10317 assert(Scale != 0 &&
"Unexpected opcode!");
10318 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
10323 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
10324 StackOffsetOperand.
setImm(NewImm);
10330 bool ShouldSignReturnAddr) {
10331 if (!ShouldSignReturnAddr)
10337 TII->get(AArch64::PAUTH_EPILOGUE))
10341void AArch64InstrInfo::buildOutlinedFrame(
10345 AArch64FunctionInfo *FI = MF.
getInfo<AArch64FunctionInfo>();
10353 unsigned TailOpcode;
10355 TailOpcode = AArch64::TCRETURNdi;
10359 TailOpcode = AArch64::TCRETURNriALL;
10370 bool IsLeafFunction =
true;
10373 auto IsNonTailCall = [](
const MachineInstr &
MI) {
10374 return MI.isCall() && !
MI.isReturn();
10384 "Can only fix up stack references once");
10385 fixupPostOutline(
MBB);
10387 IsLeafFunction =
false;
10398 Et = std::prev(
MBB.
end());
10408 if (MF.
getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
10412 CFIBuilder.buildDefCFAOffset(16);
10416 CFIBuilder.buildOffset(AArch64::LR, -16);
10457 fixupPostOutline(
MBB);
10468 .addGlobalAddress(
M.getNamedValue(MF.
getName()))
10478 .addGlobalAddress(
M.getNamedValue(MF.
getName())));
10487 MachineInstr *Save;
10488 MachineInstr *Restore;
10494 assert(
Reg &&
"No callee-saved register available?");
10528 .addGlobalAddress(
M.getNamedValue(MF.
getName())));
10536bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
10544 bool AllowSideEffects)
const {
10546 const AArch64Subtarget &STI = MF.
getSubtarget<AArch64Subtarget>();
10549 if (
TRI.isGeneralPurposeRegister(MF,
Reg)) {
10562 assert(STI.hasNEON() &&
"Expected to have NEON.");
10568std::optional<DestSourcePair>
10573 if (((
MI.getOpcode() == AArch64::ORRWrs &&
10574 MI.getOperand(1).getReg() == AArch64::WZR &&
10575 MI.getOperand(3).getImm() == 0x0) ||
10576 (
MI.getOpcode() == AArch64::ORRWrr &&
10577 MI.getOperand(1).getReg() == AArch64::WZR)) &&
10579 (!
MI.getOperand(0).getReg().isVirtual() ||
10580 MI.getOperand(0).getSubReg() == 0) &&
10581 (!
MI.getOperand(0).getReg().isPhysical() ||
10586 if (
MI.getOpcode() == AArch64::ORRXrs &&
10587 MI.getOperand(1).getReg() == AArch64::XZR &&
10588 MI.getOperand(3).getImm() == 0x0)
10591 return std::nullopt;
10594std::optional<DestSourcePair>
10596 if ((
MI.getOpcode() == AArch64::ORRWrs &&
10597 MI.getOperand(1).getReg() == AArch64::WZR &&
10598 MI.getOperand(3).getImm() == 0x0) ||
10599 (
MI.getOpcode() == AArch64::ORRWrr &&
10600 MI.getOperand(1).getReg() == AArch64::WZR))
10602 return std::nullopt;
10605std::optional<RegImmPair>
10614 return std::nullopt;
10616 switch (
MI.getOpcode()) {
10618 return std::nullopt;
10619 case AArch64::SUBWri:
10620 case AArch64::SUBXri:
10621 case AArch64::SUBSWri:
10622 case AArch64::SUBSXri:
10625 case AArch64::ADDSWri:
10626 case AArch64::ADDSXri:
10627 case AArch64::ADDWri:
10628 case AArch64::ADDXri: {
10630 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
10631 !
MI.getOperand(2).isImm())
10632 return std::nullopt;
10633 int Shift =
MI.getOperand(3).getImm();
10634 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
10638 return RegImmPair{
MI.getOperand(1).getReg(),
Offset};
10644static std::optional<ParamLoadedValue>
10648 auto DestSrc =
TII->isCopyLikeInstr(
MI);
10650 return std::nullopt;
10652 Register DestReg = DestSrc->Destination->getReg();
10653 Register SrcReg = DestSrc->Source->getReg();
10656 return std::nullopt;
10661 if (DestReg == DescribedReg)
10665 if (
MI.getOpcode() == AArch64::ORRWrs &&
10666 TRI->isSuperRegister(DestReg, DescribedReg))
10670 if (
MI.getOpcode() == AArch64::ORRXrs &&
10671 TRI->isSubRegister(DestReg, DescribedReg)) {
10672 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
10676 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
10677 "Unhandled ORR[XW]rs copy case");
10679 return std::nullopt;
10682bool AArch64InstrInfo::isFunctionSafeToSplit(
const MachineFunction &MF)
const {
10687 if (MF.
getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(
true))
10693bool AArch64InstrInfo::isMBBSafeToSplitToCold(
10697 auto isAsmGoto = [](
const MachineInstr &
MI) {
10698 return MI.getOpcode() == AArch64::INLINEASM_BR;
10708 auto containsMBB = [&
MBB](
const MachineJumpTableEntry &JTE) {
10715 for (
const MachineInstr &
MI :
MBB) {
10716 switch (
MI.getOpcode()) {
10717 case TargetOpcode::G_BRJT:
10718 case AArch64::JumpTableDest32:
10719 case AArch64::JumpTableDest16:
10720 case AArch64::JumpTableDest8:
10731std::optional<ParamLoadedValue>
10734 const MachineFunction *MF =
MI.getMF();
10736 switch (
MI.getOpcode()) {
10737 case AArch64::MOVZWi:
10738 case AArch64::MOVZXi: {
10741 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(),
Reg))
10742 return std::nullopt;
10744 if (!
MI.getOperand(1).isImm())
10745 return std::nullopt;
10746 int64_t Immediate =
MI.getOperand(1).getImm();
10747 int Shift =
MI.getOperand(2).getImm();
10751 case AArch64::ORRWrs:
10752 case AArch64::ORRXrs:
10759bool AArch64InstrInfo::isExtendLikelyToBeFolded(
10762 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
10763 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
10766 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
10770 if (!
MRI.hasOneNonDBGUse(DefReg))
10775 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
10776 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
10779uint64_t AArch64InstrInfo::getElementSizeForOpcode(
unsigned Opc)
const {
10783bool AArch64InstrInfo::isPTestLikeOpcode(
unsigned Opc)
const {
10787bool AArch64InstrInfo::isWhileOpcode(
unsigned Opc)
const {
10792AArch64InstrInfo::getTailDuplicateSize(
CodeGenOptLevel OptLevel)
const {
10796bool AArch64InstrInfo::isLegalAddressingMode(
unsigned NumBytes, int64_t
Offset,
10797 unsigned Scale)
const {
10808 unsigned Shift =
Log2_64(NumBytes);
10809 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
10817 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
10822 return AArch64::BLRNoIP;
10824 return AArch64::BLR;
10829 Register TargetReg,
bool FrameSetup)
const {
10830 assert(TargetReg != AArch64::SP &&
"New top of stack cannot already be in SP");
10842 MF.
insert(MBBInsertPoint, LoopTestMBB);
10845 MF.
insert(MBBInsertPoint, LoopBodyMBB);
10847 MF.
insert(MBBInsertPoint, ExitMBB);
10857 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
10865 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::Bcc))
10871 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
10884 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
10903 MBB.addSuccessor(LoopTestMBB);
10909 return ExitMBB->
begin();
10926 unsigned CompCounterOprNum;
10930 unsigned UpdateCounterOprNum;
10934 bool IsUpdatePriorComp;
10946 TII(MF->getSubtarget().getInstrInfo()),
10947 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
10948 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
10949 CompCounterOprNum(CompCounterOprNum), Update(Update),
10950 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
10951 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
10953 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
10959 std::optional<bool> createTripCountGreaterCondition(
10960 int TC, MachineBasicBlock &
MBB,
10961 SmallVectorImpl<MachineOperand> &CondParam)
override {
10969 void createRemainingIterationsGreaterCondition(
10970 int TC, MachineBasicBlock &
MBB, SmallVectorImpl<MachineOperand> &
Cond,
10971 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts)
override;
10973 void setPreheader(MachineBasicBlock *NewPreheader)
override {}
10975 void adjustTripCount(
int TripCountAdjust)
override {}
10977 bool isMVEExpanderSupported()
override {
return true; }
10990 MBB.getParent()->getSubtarget().getRegisterInfo();
10995 Result =
MRI.createVirtualRegister(
10998 }
else if (
I == ReplaceOprNum) {
10999 MRI.constrainRegClass(ReplaceReg,
11004 MBB.insert(InsertTo, NewMI);
11008void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
11024 assert(CondBranch->getOpcode() == AArch64::Bcc);
11028 if (CondBranch->getOperand(1).getMBB() == LoopBB)
11035 auto AccumulateCond = [&](
Register CurCond,
11037 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
11046 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
11050 for (
int I = 0;
I <= TC; ++
I) {
11056 AccCond = AccumulateCond(AccCond, CC);
11060 if (Update != Comp && IsUpdatePriorComp) {
11062 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
11063 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
11067 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
11069 }
else if (Update != Comp) {
11074 Counter = NextCounter;
11078 if (LastStage0Insts.
empty()) {
11082 if (IsUpdatePriorComp)
11087 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
11090 for (
int I = 0;
I <= TC; ++
I) {
11094 AccCond = AccumulateCond(AccCond, CC);
11095 if (
I != TC && Update != Comp)
11098 Counter = NextCounter;
11114 assert(Phi.getNumOperands() == 5);
11115 if (Phi.getOperand(2).getMBB() ==
MBB) {
11116 RegMBB = Phi.getOperand(1).getReg();
11117 RegOther = Phi.getOperand(3).getReg();
11119 assert(Phi.getOperand(4).getMBB() ==
MBB);
11120 RegMBB = Phi.getOperand(3).getReg();
11121 RegOther = Phi.getOperand(1).getReg();
11126 if (!
Reg.isVirtual())
11129 return MRI.getVRegDef(
Reg)->getParent() != BB;
11135 unsigned &UpdateCounterOprNum,
Register &InitReg,
11136 bool &IsUpdatePriorComp) {
11150 if (!
Reg.isVirtual())
11153 UpdateInst =
nullptr;
11154 UpdateCounterOprNum = 0;
11156 IsUpdatePriorComp =
true;
11160 if (Def->getParent() != LoopBB)
11162 if (Def->isCopy()) {
11164 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
11166 CurReg = Def->getOperand(1).getReg();
11167 }
else if (Def->isPHI()) {
11171 IsUpdatePriorComp =
false;
11176 switch (Def->getOpcode()) {
11177 case AArch64::ADDSXri:
11178 case AArch64::ADDSWri:
11179 case AArch64::SUBSXri:
11180 case AArch64::SUBSWri:
11181 case AArch64::ADDXri:
11182 case AArch64::ADDWri:
11183 case AArch64::SUBXri:
11184 case AArch64::SUBWri:
11186 UpdateCounterOprNum = 1;
11188 case AArch64::ADDSXrr:
11189 case AArch64::ADDSWrr:
11190 case AArch64::SUBSXrr:
11191 case AArch64::SUBSWrr:
11192 case AArch64::ADDXrr:
11193 case AArch64::ADDWrr:
11194 case AArch64::SUBXrr:
11195 case AArch64::SUBWrr:
11198 UpdateCounterOprNum = 1;
11200 UpdateCounterOprNum = 2;
11207 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
11222std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
11233 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
11244 if (
TBB == LoopBB && FBB == LoopBB)
11248 if (
TBB != LoopBB && FBB ==
nullptr)
11251 assert((
TBB == LoopBB || FBB == LoopBB) &&
11252 "The Loop must be a single-basic-block loop");
11257 if (CondBranch->
getOpcode() != AArch64::Bcc)
11265 unsigned CompCounterOprNum = 0;
11267 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
11271 switch (
MI.getOpcode()) {
11272 case AArch64::SUBSXri:
11273 case AArch64::SUBSWri:
11274 case AArch64::ADDSXri:
11275 case AArch64::ADDSWri:
11277 CompCounterOprNum = 1;
11279 case AArch64::ADDSWrr:
11280 case AArch64::ADDSXrr:
11281 case AArch64::SUBSWrr:
11282 case AArch64::SUBSXrr:
11286 if (isWhileOpcode(
MI.getOpcode())) {
11293 if (CompCounterOprNum == 0) {
11295 CompCounterOprNum = 2;
11297 CompCounterOprNum = 1;
11309 bool IsUpdatePriorComp;
11310 unsigned UpdateCounterOprNum;
11312 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
11315 return std::make_unique<AArch64PipelinerLoopInfo>(
11316 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
11326 TypeSize Scale(0U,
false), Width(0U,
false);
11327 int64_t MinOffset, MaxOffset;
11328 if (
getMemOpInfo(
MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
11330 if (
MI.getOperand(ImmIdx).isImm() && !
MI.getOperand(ImmIdx - 1).isFI()) {
11331 int64_t Imm =
MI.getOperand(ImmIdx).getImm();
11332 if (Imm < MinOffset || Imm > MaxOffset) {
11333 ErrInfo =
"Unexpected immediate on load/store instruction";
11339 const MCInstrDesc &MCID =
MI.getDesc();
11341 const MachineOperand &MO =
MI.getOperand(
Op);
11345 ErrInfo =
"OPERAND_IMPLICIT_IMM_0 should be 0";
11354 ErrInfo =
"OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
11365#define GET_INSTRINFO_HELPERS
11366#define GET_INSTRMAP_INFO
11367#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static cl::opt< unsigned > GatherOptSearchLimit("aarch64-search-limit", cl::Hidden, cl::init(2048), cl::desc("Restrict range of instructions to search for the " "machine-combiner gather pattern optimization"))
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment, StringRef RegScale={})
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, MCRegister Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static bool isPostIndexLdStOpcode(unsigned Opcode)
Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
static unsigned getBranchDisplacementBits(unsigned Opc)
static cl::opt< unsigned > CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(9), cl::desc("Restrict range of CB instructions (DEBUG)"))
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static void appendReadRegExpr(SmallVectorImpl< char > &Expr, unsigned RegNum)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool getLoadPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Search for patterns of LD instructions we can optimize.
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static void appendLoadRegExpr(SmallVectorImpl< char > &Expr, int64_t OffsetFromDefCFA)
static void appendConstantExpr(SmallVectorImpl< char > &Expr, int64_t Constant, dwarf::LocationAtom Operation)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static void generateGatherLanePattern(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned Pattern, unsigned NumLanes)
Generate optimized instruction sequence for gather load patterns to improve Memory-Level Parallelism ...
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool getGatherLanePattern(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, unsigned LoadLaneOpCode, unsigned NumLanes)
Check if the given instruction forms a gather load pattern that can be optimized for better Memory-Le...
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
Machine Check Debug Module
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
MachineInstr unsigned OpIdx
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallSet class.
This file defines the SmallVector class.
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setOutliningStyle(const std::string &Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isAsCheapAsAMove(const MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This is an important base class in LLVM.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
Instructions::iterator instr_iterator
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
LLVM_ABI bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
LLVM_ABI uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
LLVM_ABI bool isLoadFoldBarrier() const
Returns true if it is illegal to fold a load across this instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
LLVM_ABI void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
LLVM_ABI void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
LLVM_ABI MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
LLVM_ABI MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Value * getOperand(unsigned i) const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_ARM64EC_CALLMANGLE
MO_ARM64EC_CALLMANGLE - Operand refers to the Arm64EC-mangled version of a symbol,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ ScalablePredicateVector
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
LLVM_ABI Instruction & back() const
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool succeeded(LogicalResult Result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
static bool isIndirectBranchOpcode(int Opc)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
static bool isSEHInstruction(const MachineInstr &MI)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
unsigned getUndefRegState(bool B)
static MCRegister getXRegFromWReg(MCRegister Reg)
unsigned getDefRegState(bool B)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA, std::optional< int64_t > IncomingVGOffsetFromDefCFA)
unsigned getKillRegState(bool B)
FunctionAddr VTableAddr Next
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
static bool isUncondBranchOpcode(int Opc)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
void appendLEB128(SmallVectorImpl< U > &Buffer, T Value)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
static const MachineMemOperand::Flags MOStridedAccess
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
LLVM_ABI static const MBBSectionID ColdSectionID
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.