65#define GET_INSTRINFO_CTOR_DTOR
66#include "AArch64GenInstrInfo.inc"
70 cl::desc(
"Restrict range of CB instructions (DEBUG)"));
74 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
78 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
82 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
86 cl::desc(
"Restrict range of B instructions (DEBUG)"));
90 cl::desc(
"Restrict range of instructions to search for the "
91 "machine-combiner gather pattern optimization"));
96 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
107 auto Op =
MI.getOpcode();
108 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
109 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
113 if (
MI.isMetaInstruction())
118 unsigned NumBytes = 0;
122 NumBytes =
Desc.getSize() ?
Desc.getSize() : 4;
125 if (!MFI->shouldSignReturnAddress(MF))
129 auto Method = STI.getAuthenticatedLRCheckMethod(*MF);
137 switch (
Desc.getOpcode()) {
140 return Desc.getSize();
147 case TargetOpcode::STACKMAP:
150 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
152 case TargetOpcode::PATCHPOINT:
155 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
157 case TargetOpcode::STATEPOINT:
159 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
164 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
169 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
171 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
172 case TargetOpcode::PATCHABLE_TAIL_CALL:
173 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
177 case TargetOpcode::PATCHABLE_EVENT_CALL:
183 NumBytes =
MI.getOperand(1).getImm();
185 case TargetOpcode::BUNDLE:
186 NumBytes = getInstBundleLength(
MI);
193unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
197 while (++
I != E &&
I->isInsideBundle()) {
198 assert(!
I->isBundle() &&
"No nested bundle!");
233 case AArch64::CBWPri:
234 case AArch64::CBXPri:
235 case AArch64::CBWPrr:
236 case AArch64::CBXPrr:
265 case AArch64::CBWPri:
266 case AArch64::CBXPri:
267 case AArch64::CBWPrr:
268 case AArch64::CBXPrr:
274 int64_t BrOffset)
const {
276 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
277 "over conditional branch expansion");
278 return isIntN(Bits, BrOffset / 4);
283 switch (
MI.getOpcode()) {
287 return MI.getOperand(0).getMBB();
292 return MI.getOperand(2).getMBB();
298 return MI.getOperand(1).getMBB();
299 case AArch64::CBWPri:
300 case AArch64::CBXPri:
301 case AArch64::CBWPrr:
302 case AArch64::CBXPrr:
303 return MI.getOperand(3).getMBB();
313 assert(RS &&
"RegScavenger required for long branching");
315 "new block should be inserted for expanding unconditional branch");
318 "restore block should be inserted for restoring clobbered registers");
325 "Branch offsets outside of the signed 33-bit range not supported");
336 RS->enterBasicBlockEnd(
MBB);
339 constexpr Register Reg = AArch64::X16;
340 if (!RS->isRegUsed(Reg)) {
341 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
348 Register Scavenged = RS->FindUnusedReg(&AArch64::GPR64RegClass);
349 if (Scavenged != AArch64::NoRegister &&
351 buildIndirectBranch(Scavenged, NewDestBB);
352 RS->setRegUsed(Scavenged);
361 "Unable to insert indirect branch inside function that has red zone");
384 bool AllowModify)
const {
391 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
392 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
396 if (!isUnpredicatedTerminator(*
I))
403 unsigned LastOpc = LastInst->
getOpcode();
404 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
419 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
426 LastInst = SecondLastInst;
428 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
433 SecondLastInst = &*
I;
434 SecondLastOpc = SecondLastInst->
getOpcode();
445 LastInst = SecondLastInst;
447 if (
I ==
MBB.begin() || !isUnpredicatedTerminator(*--
I)) {
449 "unreachable unconditional branches removed above");
458 SecondLastInst = &*
I;
459 SecondLastOpc = SecondLastInst->
getOpcode();
463 if (SecondLastInst &&
I !=
MBB.begin() && isUnpredicatedTerminator(*--
I))
479 I->eraseFromParent();
488 I->eraseFromParent();
497 MachineBranchPredicate &MBP,
498 bool AllowModify)
const {
508 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
509 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
513 if (!isUnpredicatedTerminator(*
I))
518 unsigned LastOpc = LastInst->
getOpcode();
533 assert(MBP.TrueDest &&
"expected!");
534 MBP.FalseDest =
MBB.getNextNode();
536 MBP.ConditionDef =
nullptr;
537 MBP.SingleUseCondition =
false;
541 MBP.Predicate = (LastOpc == AArch64::CBNZX || LastOpc == AArch64::CBNZW)
542 ? MachineBranchPredicate::PRED_NE
543 : MachineBranchPredicate::PRED_EQ;
559 Cond[1].setImm(AArch64::CBNZW);
562 Cond[1].setImm(AArch64::CBZW);
565 Cond[1].setImm(AArch64::CBNZX);
568 Cond[1].setImm(AArch64::CBZX);
571 Cond[1].setImm(AArch64::TBNZW);
574 Cond[1].setImm(AArch64::TBZW);
577 Cond[1].setImm(AArch64::TBNZX);
580 Cond[1].setImm(AArch64::TBZX);
584 case AArch64::CBWPri:
585 case AArch64::CBXPri:
586 case AArch64::CBWPrr:
587 case AArch64::CBXPrr: {
600 int *BytesRemoved)
const {
610 I->eraseFromParent();
614 if (
I ==
MBB.begin()) {
627 I->eraseFromParent();
634void AArch64InstrInfo::instantiateCondBranch(
664 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
692 if (!
DefMI->isFullCopy())
694 VReg =
DefMI->getOperand(1).getReg();
703 unsigned *NewReg =
nullptr) {
708 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
712 switch (
DefMI->getOpcode()) {
713 case AArch64::SUBREG_TO_REG:
717 if (!
DefMI->getOperand(1).isImm() ||
DefMI->getOperand(1).getImm() != 0)
719 if (!
DefMI->getOperand(2).isReg())
721 if (!
DefMI->getOperand(3).isImm() ||
722 DefMI->getOperand(3).getImm() != AArch64::sub_32)
725 if (
DefMI->getOpcode() != AArch64::MOVi32imm)
727 if (!
DefMI->getOperand(1).isImm() ||
DefMI->getOperand(1).getImm() != 1)
730 SrcReg = AArch64::XZR;
731 Opc = AArch64::CSINCXr;
734 case AArch64::MOVi32imm:
735 case AArch64::MOVi64imm:
736 if (!
DefMI->getOperand(1).isImm() ||
DefMI->getOperand(1).getImm() != 1)
738 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
739 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
742 case AArch64::ADDSXri:
743 case AArch64::ADDSWri:
745 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
750 case AArch64::ADDXri:
751 case AArch64::ADDWri:
753 if (!
DefMI->getOperand(2).isImm() ||
DefMI->getOperand(2).getImm() != 1 ||
754 DefMI->getOperand(3).getImm() != 0)
756 SrcReg =
DefMI->getOperand(1).getReg();
757 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
760 case AArch64::ORNXrr:
761 case AArch64::ORNWrr: {
764 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
766 SrcReg =
DefMI->getOperand(2).getReg();
767 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
771 case AArch64::SUBSXrr:
772 case AArch64::SUBSWrr:
774 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
779 case AArch64::SUBXrr:
780 case AArch64::SUBWrr: {
783 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
785 SrcReg =
DefMI->getOperand(2).getReg();
786 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
792 assert(
Opc && SrcReg &&
"Missing parameters");
804 int &FalseCycles)
const {
808 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
815 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
819 unsigned ExtraCondLat =
Cond.size() != 1;
823 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
824 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
826 CondCycles = 1 + ExtraCondLat;
827 TrueCycles = FalseCycles = 1;
837 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
838 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
839 CondCycles = 5 + ExtraCondLat;
840 TrueCycles = FalseCycles = 2;
857 switch (
Cond.size()) {
889 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
895 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
934 unsigned SUBSOpC, SUBSDestReg;
940 case AArch64::CBWPri:
941 SUBSOpC = AArch64::SUBSWri;
942 SUBSDestReg = AArch64::WZR;
945 case AArch64::CBXPri:
946 SUBSOpC = AArch64::SUBSXri;
947 SUBSDestReg = AArch64::XZR;
950 case AArch64::CBWPrr:
951 SUBSOpC = AArch64::SUBSWrr;
952 SUBSDestReg = AArch64::WZR;
955 case AArch64::CBXPrr:
956 SUBSOpC = AArch64::SUBSXrr;
957 SUBSDestReg = AArch64::XZR;
976 bool TryFold =
false;
977 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
978 RC = &AArch64::GPR64RegClass;
979 Opc = AArch64::CSELXr;
981 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
982 RC = &AArch64::GPR32RegClass;
983 Opc = AArch64::CSELWr;
985 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
986 RC = &AArch64::FPR64RegClass;
987 Opc = AArch64::FCSELDrrr;
988 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
989 RC = &AArch64::FPR32RegClass;
990 Opc = AArch64::FCSELSrrr;
992 assert(RC &&
"Unsupported regclass");
1011 MRI.clearKillFlags(NewReg);
1016 MRI.constrainRegClass(TrueReg, RC);
1019 (FalseReg.
isVirtual() || FalseReg == AArch64::WZR ||
1020 FalseReg == AArch64::XZR) &&
1021 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1023 MRI.constrainRegClass(FalseReg, RC);
1038 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
1043 return Is.
size() <= 2;
1049 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1050 if (isExynosCheapAsMove(
MI))
1052 return MI.isAsCheapAsAMove();
1055 switch (
MI.getOpcode()) {
1057 return MI.isAsCheapAsAMove();
1059 case AArch64::ADDWrs:
1060 case AArch64::ADDXrs:
1061 case AArch64::SUBWrs:
1062 case AArch64::SUBXrs:
1063 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
1068 case AArch64::MOVi32imm:
1070 case AArch64::MOVi64imm:
1075bool AArch64InstrInfo::isFalkorShiftExtFast(
const MachineInstr &
MI) {
1076 switch (
MI.getOpcode()) {
1080 case AArch64::ADDWrs:
1081 case AArch64::ADDXrs:
1082 case AArch64::ADDSWrs:
1083 case AArch64::ADDSXrs: {
1084 unsigned Imm =
MI.getOperand(3).getImm();
1091 case AArch64::ADDWrx:
1092 case AArch64::ADDXrx:
1093 case AArch64::ADDXrx64:
1094 case AArch64::ADDSWrx:
1095 case AArch64::ADDSXrx:
1096 case AArch64::ADDSXrx64: {
1097 unsigned Imm =
MI.getOperand(3).getImm();
1109 case AArch64::SUBWrs:
1110 case AArch64::SUBSWrs: {
1111 unsigned Imm =
MI.getOperand(3).getImm();
1113 return ShiftVal == 0 ||
1117 case AArch64::SUBXrs:
1118 case AArch64::SUBSXrs: {
1119 unsigned Imm =
MI.getOperand(3).getImm();
1121 return ShiftVal == 0 ||
1125 case AArch64::SUBWrx:
1126 case AArch64::SUBXrx:
1127 case AArch64::SUBXrx64:
1128 case AArch64::SUBSWrx:
1129 case AArch64::SUBSXrx:
1130 case AArch64::SUBSXrx64: {
1131 unsigned Imm =
MI.getOperand(3).getImm();
1143 case AArch64::LDRBBroW:
1144 case AArch64::LDRBBroX:
1145 case AArch64::LDRBroW:
1146 case AArch64::LDRBroX:
1147 case AArch64::LDRDroW:
1148 case AArch64::LDRDroX:
1149 case AArch64::LDRHHroW:
1150 case AArch64::LDRHHroX:
1151 case AArch64::LDRHroW:
1152 case AArch64::LDRHroX:
1153 case AArch64::LDRQroW:
1154 case AArch64::LDRQroX:
1155 case AArch64::LDRSBWroW:
1156 case AArch64::LDRSBWroX:
1157 case AArch64::LDRSBXroW:
1158 case AArch64::LDRSBXroX:
1159 case AArch64::LDRSHWroW:
1160 case AArch64::LDRSHWroX:
1161 case AArch64::LDRSHXroW:
1162 case AArch64::LDRSHXroX:
1163 case AArch64::LDRSWroW:
1164 case AArch64::LDRSWroX:
1165 case AArch64::LDRSroW:
1166 case AArch64::LDRSroX:
1167 case AArch64::LDRWroW:
1168 case AArch64::LDRWroX:
1169 case AArch64::LDRXroW:
1170 case AArch64::LDRXroX:
1171 case AArch64::PRFMroW:
1172 case AArch64::PRFMroX:
1173 case AArch64::STRBBroW:
1174 case AArch64::STRBBroX:
1175 case AArch64::STRBroW:
1176 case AArch64::STRBroX:
1177 case AArch64::STRDroW:
1178 case AArch64::STRDroX:
1179 case AArch64::STRHHroW:
1180 case AArch64::STRHHroX:
1181 case AArch64::STRHroW:
1182 case AArch64::STRHroX:
1183 case AArch64::STRQroW:
1184 case AArch64::STRQroX:
1185 case AArch64::STRSroW:
1186 case AArch64::STRSroX:
1187 case AArch64::STRWroW:
1188 case AArch64::STRWroX:
1189 case AArch64::STRXroW:
1190 case AArch64::STRXroX: {
1191 unsigned IsSigned =
MI.getOperand(3).getImm();
1198 unsigned Opc =
MI.getOpcode();
1202 case AArch64::SEH_StackAlloc:
1203 case AArch64::SEH_SaveFPLR:
1204 case AArch64::SEH_SaveFPLR_X:
1205 case AArch64::SEH_SaveReg:
1206 case AArch64::SEH_SaveReg_X:
1207 case AArch64::SEH_SaveRegP:
1208 case AArch64::SEH_SaveRegP_X:
1209 case AArch64::SEH_SaveFReg:
1210 case AArch64::SEH_SaveFReg_X:
1211 case AArch64::SEH_SaveFRegP:
1212 case AArch64::SEH_SaveFRegP_X:
1213 case AArch64::SEH_SetFP:
1214 case AArch64::SEH_AddFP:
1215 case AArch64::SEH_Nop:
1216 case AArch64::SEH_PrologEnd:
1217 case AArch64::SEH_EpilogStart:
1218 case AArch64::SEH_EpilogEnd:
1219 case AArch64::SEH_PACSignLR:
1220 case AArch64::SEH_SaveAnyRegQP:
1221 case AArch64::SEH_SaveAnyRegQPX:
1222 case AArch64::SEH_AllocZ:
1223 case AArch64::SEH_SaveZReg:
1224 case AArch64::SEH_SavePReg:
1231 unsigned &SubIdx)
const {
1232 switch (
MI.getOpcode()) {
1235 case AArch64::SBFMXri:
1236 case AArch64::UBFMXri:
1239 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1242 SrcReg =
MI.getOperand(1).getReg();
1243 DstReg =
MI.getOperand(0).getReg();
1244 SubIdx = AArch64::sub_32;
1253 int64_t OffsetA = 0, OffsetB = 0;
1254 TypeSize WidthA(0,
false), WidthB(0,
false);
1255 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1276 OffsetAIsScalable == OffsetBIsScalable) {
1277 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1278 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1279 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1280 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1298 switch (
MI.getOpcode()) {
1301 if (
MI.getOperand(0).getImm() == 0x14)
1308 case AArch64::MSRpstatesvcrImm1:
1315 auto Next = std::next(
MI.getIterator());
1316 return Next !=
MBB->end() &&
Next->isCFIInstruction();
1323 Register &SrcReg2, int64_t &CmpMask,
1324 int64_t &CmpValue)
const {
1328 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1329 if (!
MI.getOperand(1).isReg() ||
MI.getOperand(1).getSubReg())
1332 switch (
MI.getOpcode()) {
1335 case AArch64::PTEST_PP:
1336 case AArch64::PTEST_PP_ANY:
1337 case AArch64::PTEST_PP_FIRST:
1338 SrcReg =
MI.getOperand(0).getReg();
1339 SrcReg2 =
MI.getOperand(1).getReg();
1340 if (
MI.getOperand(2).getSubReg())
1347 case AArch64::SUBSWrr:
1348 case AArch64::SUBSWrs:
1349 case AArch64::SUBSWrx:
1350 case AArch64::SUBSXrr:
1351 case AArch64::SUBSXrs:
1352 case AArch64::SUBSXrx:
1353 case AArch64::ADDSWrr:
1354 case AArch64::ADDSWrs:
1355 case AArch64::ADDSWrx:
1356 case AArch64::ADDSXrr:
1357 case AArch64::ADDSXrs:
1358 case AArch64::ADDSXrx:
1360 SrcReg =
MI.getOperand(1).getReg();
1361 SrcReg2 =
MI.getOperand(2).getReg();
1364 if (
MI.getOperand(2).getSubReg())
1370 case AArch64::SUBSWri:
1371 case AArch64::ADDSWri:
1372 case AArch64::SUBSXri:
1373 case AArch64::ADDSXri:
1374 SrcReg =
MI.getOperand(1).getReg();
1377 CmpValue =
MI.getOperand(2).getImm();
1379 case AArch64::ANDSWri:
1380 case AArch64::ANDSXri:
1383 SrcReg =
MI.getOperand(1).getReg();
1387 MI.getOperand(2).getImm(),
1388 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1397 assert(
MBB &&
"Can't get MachineBasicBlock here");
1399 assert(MF &&
"Can't get MachineFunction here");
1404 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands();
OpIdx < EndIdx;
1411 if (!OpRegCstraints)
1419 "Operand has register constraints without being a register!");
1422 if (
Reg.isPhysical()) {
1426 !
MRI->constrainRegClass(
Reg, OpRegCstraints))
1439 bool MIDefinesZeroReg =
false;
1440 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1441 MI.definesRegister(AArch64::XZR,
nullptr))
1442 MIDefinesZeroReg =
true;
1444 switch (
MI.getOpcode()) {
1446 return MI.getOpcode();
1447 case AArch64::ADDSWrr:
1448 return AArch64::ADDWrr;
1449 case AArch64::ADDSWri:
1450 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1451 case AArch64::ADDSWrs:
1452 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1453 case AArch64::ADDSWrx:
1454 return AArch64::ADDWrx;
1455 case AArch64::ADDSXrr:
1456 return AArch64::ADDXrr;
1457 case AArch64::ADDSXri:
1458 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1459 case AArch64::ADDSXrs:
1460 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1461 case AArch64::ADDSXrx:
1462 return AArch64::ADDXrx;
1463 case AArch64::SUBSWrr:
1464 return AArch64::SUBWrr;
1465 case AArch64::SUBSWri:
1466 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1467 case AArch64::SUBSWrs:
1468 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1469 case AArch64::SUBSWrx:
1470 return AArch64::SUBWrx;
1471 case AArch64::SUBSXrr:
1472 return AArch64::SUBXrr;
1473 case AArch64::SUBSXri:
1474 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1475 case AArch64::SUBSXrs:
1476 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1477 case AArch64::SUBSXrx:
1478 return AArch64::SUBXrx;
1493 if (To == To->getParent()->begin())
1498 if (To->getParent() != From->getParent())
1510 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1511 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1517std::optional<unsigned>
1521 unsigned MaskOpcode =
Mask->getOpcode();
1522 unsigned PredOpcode = Pred->
getOpcode();
1523 bool PredIsPTestLike = isPTestLikeOpcode(PredOpcode);
1524 bool PredIsWhileLike = isWhileOpcode(PredOpcode);
1526 if (PredIsWhileLike) {
1530 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1537 getElementSizeForOpcode(MaskOpcode) ==
1538 getElementSizeForOpcode(PredOpcode))
1544 if (PTest->
getOpcode() == AArch64::PTEST_PP_FIRST &&
1551 if (PredIsPTestLike) {
1556 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1564 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1565 PTestLikeMask->getOperand(1).getReg().isVirtual())
1567 MRI->getUniqueVRegDef(PTestLikeMask->getOperand(1).getReg());
1573 getElementSizeForOpcode(MaskOpcode) ==
1574 getElementSizeForOpcode(PredOpcode)) {
1575 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1601 uint64_t PredElementSize = getElementSizeForOpcode(PredOpcode);
1603 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1611 switch (PredOpcode) {
1612 case AArch64::AND_PPzPP:
1613 case AArch64::BIC_PPzPP:
1614 case AArch64::EOR_PPzPP:
1615 case AArch64::NAND_PPzPP:
1616 case AArch64::NOR_PPzPP:
1617 case AArch64::ORN_PPzPP:
1618 case AArch64::ORR_PPzPP:
1619 case AArch64::BRKA_PPzP:
1620 case AArch64::BRKPA_PPzPP:
1621 case AArch64::BRKB_PPzP:
1622 case AArch64::BRKPB_PPzPP:
1623 case AArch64::RDFFR_PPz: {
1627 if (Mask != PredMask)
1631 case AArch64::BRKN_PPzP: {
1635 if ((MaskOpcode != AArch64::PTRUE_B) ||
1636 (
Mask->getOperand(1).getImm() != 31))
1640 case AArch64::PTRUE_B:
1653bool AArch64InstrInfo::optimizePTestInstr(
1654 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1656 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1657 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1659 if (Pred->
isCopy() && PTest->
getOpcode() == AArch64::PTEST_PP_FIRST) {
1663 if (
Op.isReg() &&
Op.getReg().isVirtual() &&
1664 Op.getSubReg() == AArch64::psub0)
1665 Pred =
MRI->getUniqueVRegDef(
Op.getReg());
1668 unsigned PredOpcode = Pred->
getOpcode();
1669 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1685 if (*NewOp != PredOpcode) {
1696 for (; i !=
e; ++i) {
1727 if (DeadNZCVIdx != -1) {
1746 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1747 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY ||
1748 CmpInstr.
getOpcode() == AArch64::PTEST_PP_FIRST)
1749 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1758 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1760 return (CmpValue == 0 || CmpValue == 1) &&
1761 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1769 switch (Instr.getOpcode()) {
1771 return AArch64::INSTRUCTION_LIST_END;
1773 case AArch64::ADDSWrr:
1774 case AArch64::ADDSWri:
1775 case AArch64::ADDSXrr:
1776 case AArch64::ADDSXri:
1777 case AArch64::SUBSWrr:
1778 case AArch64::SUBSWri:
1779 case AArch64::SUBSXrr:
1780 case AArch64::SUBSXri:
1781 return Instr.getOpcode();
1783 case AArch64::ADDWrr:
1784 return AArch64::ADDSWrr;
1785 case AArch64::ADDWri:
1786 return AArch64::ADDSWri;
1787 case AArch64::ADDXrr:
1788 return AArch64::ADDSXrr;
1789 case AArch64::ADDXri:
1790 return AArch64::ADDSXri;
1791 case AArch64::ADCWr:
1792 return AArch64::ADCSWr;
1793 case AArch64::ADCXr:
1794 return AArch64::ADCSXr;
1795 case AArch64::SUBWrr:
1796 return AArch64::SUBSWrr;
1797 case AArch64::SUBWri:
1798 return AArch64::SUBSWri;
1799 case AArch64::SUBXrr:
1800 return AArch64::SUBSXrr;
1801 case AArch64::SUBXri:
1802 return AArch64::SUBSXri;
1803 case AArch64::SBCWr:
1804 return AArch64::SBCSWr;
1805 case AArch64::SBCXr:
1806 return AArch64::SBCSXr;
1807 case AArch64::ANDWri:
1808 return AArch64::ANDSWri;
1809 case AArch64::ANDXri:
1810 return AArch64::ANDSXri;
1816 for (
auto *BB :
MBB->successors())
1817 if (BB->isLiveIn(AArch64::NZCV))
1826 switch (Instr.getOpcode()) {
1830 case AArch64::Bcc: {
1831 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1836 case AArch64::CSINVWr:
1837 case AArch64::CSINVXr:
1838 case AArch64::CSINCWr:
1839 case AArch64::CSINCXr:
1840 case AArch64::CSELWr:
1841 case AArch64::CSELXr:
1842 case AArch64::CSNEGWr:
1843 case AArch64::CSNEGXr:
1844 case AArch64::FCSELSrrr:
1845 case AArch64::FCSELDrrr: {
1846 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1859 Instr.getOperand(CCIdx).
getImm())
1912std::optional<UsedNZCV>
1917 if (
MI.getParent() != CmpParent)
1918 return std::nullopt;
1921 return std::nullopt;
1926 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1929 return std::nullopt;
1934 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1937 return NZCVUsedAfterCmp;
1941 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1945 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1967 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1973 "Caller guarantees that CmpInstr compares with constant 0");
1976 if (!NZVCUsed || NZVCUsed->C)
1998bool AArch64InstrInfo::substituteCmpToZero(
2002 MachineInstr *
MI =
MRI.getUniqueVRegDef(SrcReg);
2009 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2016 MI->setDesc(
get(NewOpc));
2021 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
2033 assert((CmpValue == 0 || CmpValue == 1) &&
2034 "Only comparisons to 0 or 1 considered for removal!");
2037 unsigned MIOpc =
MI.getOpcode();
2038 if (MIOpc == AArch64::CSINCWr) {
2039 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
2040 MI.getOperand(2).getReg() != AArch64::WZR)
2042 }
else if (MIOpc == AArch64::CSINCXr) {
2043 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
2044 MI.getOperand(2).getReg() != AArch64::XZR)
2054 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
2058 const unsigned CmpOpcode = CmpInstr.
getOpcode();
2060 if (CmpValue && !IsSubsRegImm)
2062 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
2067 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
2070 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2074 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2077 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
2078 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
2081 if (MIUsedNZCV.
N && !CmpValue)
2123bool AArch64InstrInfo::removeCmpToZeroOrOne(
2126 MachineInstr *
MI =
MRI.getUniqueVRegDef(SrcReg);
2130 SmallVector<MachineInstr *, 4> CCUseInstrs;
2131 bool IsInvertCC =
false;
2139 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2141 assert(Idx >= 0 &&
"Unexpected instruction using CC.");
2142 MachineOperand &CCOperand = CCUseInstr->getOperand(Idx);
2151bool AArch64InstrInfo::expandPostRAPseudo(
MachineInstr &
MI)
const {
2152 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2153 MI.getOpcode() != AArch64::CATCHRET)
2156 MachineBasicBlock &
MBB = *
MI.getParent();
2158 auto TRI = Subtarget.getRegisterInfo();
2161 if (
MI.getOpcode() == AArch64::CATCHRET) {
2163 const TargetInstrInfo *
TII =
2165 MachineBasicBlock *TargetMBB =
MI.getOperand(0).getMBB();
2170 FirstEpilogSEH = std::prev(FirstEpilogSEH);
2172 FirstEpilogSEH = std::next(FirstEpilogSEH);
2187 if (
M.getStackProtectorGuard() ==
"sysreg") {
2188 const AArch64SysReg::SysReg *SrcReg =
2189 AArch64SysReg::lookupSysRegByName(
M.getStackProtectorGuardReg());
2197 int Offset =
M.getStackProtectorGuardOffset();
2248 const GlobalValue *GV =
2251 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2257 if (Subtarget.isTargetILP32()) {
2258 unsigned Reg32 =
TRI->getSubReg(
Reg, AArch64::sub_32);
2272 assert(!Subtarget.isTargetILP32() &&
"how can large exist in ILP32?");
2299 if (Subtarget.isTargetILP32()) {
2300 unsigned Reg32 =
TRI->getSubReg(
Reg, AArch64::sub_32);
2323 switch (
MI.getOpcode()) {
2326 case AArch64::MOVZWi:
2327 case AArch64::MOVZXi:
2328 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2329 assert(
MI.getDesc().getNumOperands() == 3 &&
2330 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2334 case AArch64::ANDWri:
2335 return MI.getOperand(1).getReg() == AArch64::WZR;
2336 case AArch64::ANDXri:
2337 return MI.getOperand(1).getReg() == AArch64::XZR;
2338 case TargetOpcode::COPY:
2339 return MI.getOperand(1).getReg() == AArch64::WZR;
2347 switch (
MI.getOpcode()) {
2350 case TargetOpcode::COPY: {
2353 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2354 AArch64::GPR64RegClass.
contains(DstReg));
2356 case AArch64::ORRXrs:
2357 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2358 assert(
MI.getDesc().getNumOperands() == 4 &&
2359 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2363 case AArch64::ADDXri:
2364 if (
MI.getOperand(2).getImm() == 0) {
2365 assert(
MI.getDesc().getNumOperands() == 4 &&
2366 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2377 switch (
MI.getOpcode()) {
2380 case TargetOpcode::COPY: {
2382 return AArch64::FPR128RegClass.contains(DstReg);
2384 case AArch64::ORRv16i8:
2385 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2386 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2387 "invalid ORRv16i8 operands");
2396 int &FrameIndex)
const {
2397 switch (
MI.getOpcode()) {
2400 case AArch64::LDRWui:
2401 case AArch64::LDRXui:
2402 case AArch64::LDRBui:
2403 case AArch64::LDRHui:
2404 case AArch64::LDRSui:
2405 case AArch64::LDRDui:
2406 case AArch64::LDRQui:
2407 case AArch64::LDR_PXI:
2408 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2409 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2410 FrameIndex =
MI.getOperand(1).getIndex();
2411 return MI.getOperand(0).getReg();
2420 int &FrameIndex)
const {
2421 switch (
MI.getOpcode()) {
2424 case AArch64::STRWui:
2425 case AArch64::STRXui:
2426 case AArch64::STRBui:
2427 case AArch64::STRHui:
2428 case AArch64::STRSui:
2429 case AArch64::STRDui:
2430 case AArch64::STRQui:
2431 case AArch64::STR_PXI:
2432 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2433 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2434 FrameIndex =
MI.getOperand(1).getIndex();
2435 return MI.getOperand(0).getReg();
2445 return MMO->getFlags() & MOSuppressPair;
2451 if (
MI.memoperands_empty())
2459 return MMO->getFlags() & MOStridedAccess;
2467 case AArch64::STURSi:
2468 case AArch64::STRSpre:
2469 case AArch64::STURDi:
2470 case AArch64::STRDpre:
2471 case AArch64::STURQi:
2472 case AArch64::STRQpre:
2473 case AArch64::STURBBi:
2474 case AArch64::STURHHi:
2475 case AArch64::STURWi:
2476 case AArch64::STRWpre:
2477 case AArch64::STURXi:
2478 case AArch64::STRXpre:
2479 case AArch64::LDURSi:
2480 case AArch64::LDRSpre:
2481 case AArch64::LDURDi:
2482 case AArch64::LDRDpre:
2483 case AArch64::LDURQi:
2484 case AArch64::LDRQpre:
2485 case AArch64::LDURWi:
2486 case AArch64::LDRWpre:
2487 case AArch64::LDURXi:
2488 case AArch64::LDRXpre:
2489 case AArch64::LDRSWpre:
2490 case AArch64::LDURSWi:
2491 case AArch64::LDURHHi:
2492 case AArch64::LDURBBi:
2493 case AArch64::LDURSBWi:
2494 case AArch64::LDURSHWi:
2502 case AArch64::PRFMui:
return AArch64::PRFUMi;
2503 case AArch64::LDRXui:
return AArch64::LDURXi;
2504 case AArch64::LDRWui:
return AArch64::LDURWi;
2505 case AArch64::LDRBui:
return AArch64::LDURBi;
2506 case AArch64::LDRHui:
return AArch64::LDURHi;
2507 case AArch64::LDRSui:
return AArch64::LDURSi;
2508 case AArch64::LDRDui:
return AArch64::LDURDi;
2509 case AArch64::LDRQui:
return AArch64::LDURQi;
2510 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2511 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2512 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2513 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2514 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2515 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2516 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2517 case AArch64::STRXui:
return AArch64::STURXi;
2518 case AArch64::STRWui:
return AArch64::STURWi;
2519 case AArch64::STRBui:
return AArch64::STURBi;
2520 case AArch64::STRHui:
return AArch64::STURHi;
2521 case AArch64::STRSui:
return AArch64::STURSi;
2522 case AArch64::STRDui:
return AArch64::STURDi;
2523 case AArch64::STRQui:
return AArch64::STURQi;
2524 case AArch64::STRBBui:
return AArch64::STURBBi;
2525 case AArch64::STRHHui:
return AArch64::STURHHi;
2534 case AArch64::LDAPURBi:
2535 case AArch64::LDAPURHi:
2536 case AArch64::LDAPURi:
2537 case AArch64::LDAPURSBWi:
2538 case AArch64::LDAPURSBXi:
2539 case AArch64::LDAPURSHWi:
2540 case AArch64::LDAPURSHXi:
2541 case AArch64::LDAPURSWi:
2542 case AArch64::LDAPURXi:
2543 case AArch64::LDR_PPXI:
2544 case AArch64::LDR_PXI:
2545 case AArch64::LDR_ZXI:
2546 case AArch64::LDR_ZZXI:
2547 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
2548 case AArch64::LDR_ZZZXI:
2549 case AArch64::LDR_ZZZZXI:
2550 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
2551 case AArch64::LDRBBui:
2552 case AArch64::LDRBui:
2553 case AArch64::LDRDui:
2554 case AArch64::LDRHHui:
2555 case AArch64::LDRHui:
2556 case AArch64::LDRQui:
2557 case AArch64::LDRSBWui:
2558 case AArch64::LDRSBXui:
2559 case AArch64::LDRSHWui:
2560 case AArch64::LDRSHXui:
2561 case AArch64::LDRSui:
2562 case AArch64::LDRSWui:
2563 case AArch64::LDRWui:
2564 case AArch64::LDRXui:
2565 case AArch64::LDURBBi:
2566 case AArch64::LDURBi:
2567 case AArch64::LDURDi:
2568 case AArch64::LDURHHi:
2569 case AArch64::LDURHi:
2570 case AArch64::LDURQi:
2571 case AArch64::LDURSBWi:
2572 case AArch64::LDURSBXi:
2573 case AArch64::LDURSHWi:
2574 case AArch64::LDURSHXi:
2575 case AArch64::LDURSi:
2576 case AArch64::LDURSWi:
2577 case AArch64::LDURWi:
2578 case AArch64::LDURXi:
2579 case AArch64::PRFMui:
2580 case AArch64::PRFUMi:
2581 case AArch64::ST2Gi:
2583 case AArch64::STLURBi:
2584 case AArch64::STLURHi:
2585 case AArch64::STLURWi:
2586 case AArch64::STLURXi:
2587 case AArch64::StoreSwiftAsyncContext:
2588 case AArch64::STR_PPXI:
2589 case AArch64::STR_PXI:
2590 case AArch64::STR_ZXI:
2591 case AArch64::STR_ZZXI:
2592 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
2593 case AArch64::STR_ZZZXI:
2594 case AArch64::STR_ZZZZXI:
2595 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
2596 case AArch64::STRBBui:
2597 case AArch64::STRBui:
2598 case AArch64::STRDui:
2599 case AArch64::STRHHui:
2600 case AArch64::STRHui:
2601 case AArch64::STRQui:
2602 case AArch64::STRSui:
2603 case AArch64::STRWui:
2604 case AArch64::STRXui:
2605 case AArch64::STURBBi:
2606 case AArch64::STURBi:
2607 case AArch64::STURDi:
2608 case AArch64::STURHHi:
2609 case AArch64::STURHi:
2610 case AArch64::STURQi:
2611 case AArch64::STURSi:
2612 case AArch64::STURWi:
2613 case AArch64::STURXi:
2614 case AArch64::STZ2Gi:
2615 case AArch64::STZGi:
2616 case AArch64::TAGPstack:
2618 case AArch64::LD1B_D_IMM:
2619 case AArch64::LD1B_H_IMM:
2620 case AArch64::LD1B_IMM:
2621 case AArch64::LD1B_S_IMM:
2622 case AArch64::LD1D_IMM:
2623 case AArch64::LD1H_D_IMM:
2624 case AArch64::LD1H_IMM:
2625 case AArch64::LD1H_S_IMM:
2626 case AArch64::LD1RB_D_IMM:
2627 case AArch64::LD1RB_H_IMM:
2628 case AArch64::LD1RB_IMM:
2629 case AArch64::LD1RB_S_IMM:
2630 case AArch64::LD1RD_IMM:
2631 case AArch64::LD1RH_D_IMM:
2632 case AArch64::LD1RH_IMM:
2633 case AArch64::LD1RH_S_IMM:
2634 case AArch64::LD1RSB_D_IMM:
2635 case AArch64::LD1RSB_H_IMM:
2636 case AArch64::LD1RSB_S_IMM:
2637 case AArch64::LD1RSH_D_IMM:
2638 case AArch64::LD1RSH_S_IMM:
2639 case AArch64::LD1RSW_IMM:
2640 case AArch64::LD1RW_D_IMM:
2641 case AArch64::LD1RW_IMM:
2642 case AArch64::LD1SB_D_IMM:
2643 case AArch64::LD1SB_H_IMM:
2644 case AArch64::LD1SB_S_IMM:
2645 case AArch64::LD1SH_D_IMM:
2646 case AArch64::LD1SH_S_IMM:
2647 case AArch64::LD1SW_D_IMM:
2648 case AArch64::LD1W_D_IMM:
2649 case AArch64::LD1W_IMM:
2650 case AArch64::LD2B_IMM:
2651 case AArch64::LD2D_IMM:
2652 case AArch64::LD2H_IMM:
2653 case AArch64::LD2W_IMM:
2654 case AArch64::LD3B_IMM:
2655 case AArch64::LD3D_IMM:
2656 case AArch64::LD3H_IMM:
2657 case AArch64::LD3W_IMM:
2658 case AArch64::LD4B_IMM:
2659 case AArch64::LD4D_IMM:
2660 case AArch64::LD4H_IMM:
2661 case AArch64::LD4W_IMM:
2663 case AArch64::LDNF1B_D_IMM:
2664 case AArch64::LDNF1B_H_IMM:
2665 case AArch64::LDNF1B_IMM:
2666 case AArch64::LDNF1B_S_IMM:
2667 case AArch64::LDNF1D_IMM:
2668 case AArch64::LDNF1H_D_IMM:
2669 case AArch64::LDNF1H_IMM:
2670 case AArch64::LDNF1H_S_IMM:
2671 case AArch64::LDNF1SB_D_IMM:
2672 case AArch64::LDNF1SB_H_IMM:
2673 case AArch64::LDNF1SB_S_IMM:
2674 case AArch64::LDNF1SH_D_IMM:
2675 case AArch64::LDNF1SH_S_IMM:
2676 case AArch64::LDNF1SW_D_IMM:
2677 case AArch64::LDNF1W_D_IMM:
2678 case AArch64::LDNF1W_IMM:
2679 case AArch64::LDNPDi:
2680 case AArch64::LDNPQi:
2681 case AArch64::LDNPSi:
2682 case AArch64::LDNPWi:
2683 case AArch64::LDNPXi:
2684 case AArch64::LDNT1B_ZRI:
2685 case AArch64::LDNT1D_ZRI:
2686 case AArch64::LDNT1H_ZRI:
2687 case AArch64::LDNT1W_ZRI:
2688 case AArch64::LDPDi:
2689 case AArch64::LDPQi:
2690 case AArch64::LDPSi:
2691 case AArch64::LDPWi:
2692 case AArch64::LDPXi:
2693 case AArch64::LDRBBpost:
2694 case AArch64::LDRBBpre:
2695 case AArch64::LDRBpost:
2696 case AArch64::LDRBpre:
2697 case AArch64::LDRDpost:
2698 case AArch64::LDRDpre:
2699 case AArch64::LDRHHpost:
2700 case AArch64::LDRHHpre:
2701 case AArch64::LDRHpost:
2702 case AArch64::LDRHpre:
2703 case AArch64::LDRQpost:
2704 case AArch64::LDRQpre:
2705 case AArch64::LDRSpost:
2706 case AArch64::LDRSpre:
2707 case AArch64::LDRWpost:
2708 case AArch64::LDRWpre:
2709 case AArch64::LDRXpost:
2710 case AArch64::LDRXpre:
2711 case AArch64::ST1B_D_IMM:
2712 case AArch64::ST1B_H_IMM:
2713 case AArch64::ST1B_IMM:
2714 case AArch64::ST1B_S_IMM:
2715 case AArch64::ST1D_IMM:
2716 case AArch64::ST1H_D_IMM:
2717 case AArch64::ST1H_IMM:
2718 case AArch64::ST1H_S_IMM:
2719 case AArch64::ST1W_D_IMM:
2720 case AArch64::ST1W_IMM:
2721 case AArch64::ST2B_IMM:
2722 case AArch64::ST2D_IMM:
2723 case AArch64::ST2H_IMM:
2724 case AArch64::ST2W_IMM:
2725 case AArch64::ST3B_IMM:
2726 case AArch64::ST3D_IMM:
2727 case AArch64::ST3H_IMM:
2728 case AArch64::ST3W_IMM:
2729 case AArch64::ST4B_IMM:
2730 case AArch64::ST4D_IMM:
2731 case AArch64::ST4H_IMM:
2732 case AArch64::ST4W_IMM:
2733 case AArch64::STGPi:
2734 case AArch64::STGPreIndex:
2735 case AArch64::STZGPreIndex:
2736 case AArch64::ST2GPreIndex:
2737 case AArch64::STZ2GPreIndex:
2738 case AArch64::STGPostIndex:
2739 case AArch64::STZGPostIndex:
2740 case AArch64::ST2GPostIndex:
2741 case AArch64::STZ2GPostIndex:
2742 case AArch64::STNPDi:
2743 case AArch64::STNPQi:
2744 case AArch64::STNPSi:
2745 case AArch64::STNPWi:
2746 case AArch64::STNPXi:
2747 case AArch64::STNT1B_ZRI:
2748 case AArch64::STNT1D_ZRI:
2749 case AArch64::STNT1H_ZRI:
2750 case AArch64::STNT1W_ZRI:
2751 case AArch64::STPDi:
2752 case AArch64::STPQi:
2753 case AArch64::STPSi:
2754 case AArch64::STPWi:
2755 case AArch64::STPXi:
2756 case AArch64::STRBBpost:
2757 case AArch64::STRBBpre:
2758 case AArch64::STRBpost:
2759 case AArch64::STRBpre:
2760 case AArch64::STRDpost:
2761 case AArch64::STRDpre:
2762 case AArch64::STRHHpost:
2763 case AArch64::STRHHpre:
2764 case AArch64::STRHpost:
2765 case AArch64::STRHpre:
2766 case AArch64::STRQpost:
2767 case AArch64::STRQpre:
2768 case AArch64::STRSpost:
2769 case AArch64::STRSpre:
2770 case AArch64::STRWpost:
2771 case AArch64::STRWpre:
2772 case AArch64::STRXpost:
2773 case AArch64::STRXpre:
2775 case AArch64::LDPDpost:
2776 case AArch64::LDPDpre:
2777 case AArch64::LDPQpost:
2778 case AArch64::LDPQpre:
2779 case AArch64::LDPSpost:
2780 case AArch64::LDPSpre:
2781 case AArch64::LDPWpost:
2782 case AArch64::LDPWpre:
2783 case AArch64::LDPXpost:
2784 case AArch64::LDPXpre:
2785 case AArch64::STGPpre:
2786 case AArch64::STGPpost:
2787 case AArch64::STPDpost:
2788 case AArch64::STPDpre:
2789 case AArch64::STPQpost:
2790 case AArch64::STPQpre:
2791 case AArch64::STPSpost:
2792 case AArch64::STPSpre:
2793 case AArch64::STPWpost:
2794 case AArch64::STPWpre:
2795 case AArch64::STPXpost:
2796 case AArch64::STPXpre:
2802 switch (
MI.getOpcode()) {
2806 case AArch64::STRSui:
2807 case AArch64::STRDui:
2808 case AArch64::STRQui:
2809 case AArch64::STRXui:
2810 case AArch64::STRWui:
2811 case AArch64::LDRSui:
2812 case AArch64::LDRDui:
2813 case AArch64::LDRQui:
2814 case AArch64::LDRXui:
2815 case AArch64::LDRWui:
2816 case AArch64::LDRSWui:
2818 case AArch64::STURSi:
2819 case AArch64::STRSpre:
2820 case AArch64::STURDi:
2821 case AArch64::STRDpre:
2822 case AArch64::STURQi:
2823 case AArch64::STRQpre:
2824 case AArch64::STURWi:
2825 case AArch64::STRWpre:
2826 case AArch64::STURXi:
2827 case AArch64::STRXpre:
2828 case AArch64::LDURSi:
2829 case AArch64::LDRSpre:
2830 case AArch64::LDURDi:
2831 case AArch64::LDRDpre:
2832 case AArch64::LDURQi:
2833 case AArch64::LDRQpre:
2834 case AArch64::LDURWi:
2835 case AArch64::LDRWpre:
2836 case AArch64::LDURXi:
2837 case AArch64::LDRXpre:
2838 case AArch64::LDURSWi:
2839 case AArch64::LDRSWpre:
2841 case AArch64::LDR_ZXI:
2842 case AArch64::STR_ZXI:
2848 switch (
MI.getOpcode()) {
2851 "Unexpected instruction - was a new tail call opcode introduced?");
2853 case AArch64::TCRETURNdi:
2854 case AArch64::TCRETURNri:
2855 case AArch64::TCRETURNrix16x17:
2856 case AArch64::TCRETURNrix17:
2857 case AArch64::TCRETURNrinotx16:
2858 case AArch64::TCRETURNriALL:
2859 case AArch64::AUTH_TCRETURN:
2860 case AArch64::AUTH_TCRETURN_BTI:
2870 case AArch64::ADDWri:
2871 return AArch64::ADDSWri;
2872 case AArch64::ADDWrr:
2873 return AArch64::ADDSWrr;
2874 case AArch64::ADDWrs:
2875 return AArch64::ADDSWrs;
2876 case AArch64::ADDWrx:
2877 return AArch64::ADDSWrx;
2878 case AArch64::ANDWri:
2879 return AArch64::ANDSWri;
2880 case AArch64::ANDWrr:
2881 return AArch64::ANDSWrr;
2882 case AArch64::ANDWrs:
2883 return AArch64::ANDSWrs;
2884 case AArch64::BICWrr:
2885 return AArch64::BICSWrr;
2886 case AArch64::BICWrs:
2887 return AArch64::BICSWrs;
2888 case AArch64::SUBWri:
2889 return AArch64::SUBSWri;
2890 case AArch64::SUBWrr:
2891 return AArch64::SUBSWrr;
2892 case AArch64::SUBWrs:
2893 return AArch64::SUBSWrs;
2894 case AArch64::SUBWrx:
2895 return AArch64::SUBSWrx;
2897 case AArch64::ADDXri:
2898 return AArch64::ADDSXri;
2899 case AArch64::ADDXrr:
2900 return AArch64::ADDSXrr;
2901 case AArch64::ADDXrs:
2902 return AArch64::ADDSXrs;
2903 case AArch64::ADDXrx:
2904 return AArch64::ADDSXrx;
2905 case AArch64::ANDXri:
2906 return AArch64::ANDSXri;
2907 case AArch64::ANDXrr:
2908 return AArch64::ANDSXrr;
2909 case AArch64::ANDXrs:
2910 return AArch64::ANDSXrs;
2911 case AArch64::BICXrr:
2912 return AArch64::BICSXrr;
2913 case AArch64::BICXrs:
2914 return AArch64::BICSXrs;
2915 case AArch64::SUBXri:
2916 return AArch64::SUBSXri;
2917 case AArch64::SUBXrr:
2918 return AArch64::SUBSXrr;
2919 case AArch64::SUBXrs:
2920 return AArch64::SUBSXrs;
2921 case AArch64::SUBXrx:
2922 return AArch64::SUBSXrx;
2924 case AArch64::AND_PPzPP:
2925 return AArch64::ANDS_PPzPP;
2926 case AArch64::BIC_PPzPP:
2927 return AArch64::BICS_PPzPP;
2928 case AArch64::EOR_PPzPP:
2929 return AArch64::EORS_PPzPP;
2930 case AArch64::NAND_PPzPP:
2931 return AArch64::NANDS_PPzPP;
2932 case AArch64::NOR_PPzPP:
2933 return AArch64::NORS_PPzPP;
2934 case AArch64::ORN_PPzPP:
2935 return AArch64::ORNS_PPzPP;
2936 case AArch64::ORR_PPzPP:
2937 return AArch64::ORRS_PPzPP;
2938 case AArch64::BRKA_PPzP:
2939 return AArch64::BRKAS_PPzP;
2940 case AArch64::BRKPA_PPzPP:
2941 return AArch64::BRKPAS_PPzPP;
2942 case AArch64::BRKB_PPzP:
2943 return AArch64::BRKBS_PPzP;
2944 case AArch64::BRKPB_PPzPP:
2945 return AArch64::BRKPBS_PPzPP;
2946 case AArch64::BRKN_PPzP:
2947 return AArch64::BRKNS_PPzP;
2948 case AArch64::RDFFR_PPz:
2949 return AArch64::RDFFRS_PPz;
2950 case AArch64::PTRUE_B:
2951 return AArch64::PTRUES_B;
2962 if (
MI.hasOrderedMemoryRef())
2967 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2968 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2969 "Expected a reg or frame index operand.");
2973 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2975 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2988 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2989 Register BaseReg =
MI.getOperand(1).getReg();
2991 if (
MI.modifiesRegister(BaseReg,
TRI))
2997 switch (
MI.getOpcode()) {
3000 case AArch64::LDR_ZXI:
3001 case AArch64::STR_ZXI:
3002 if (!Subtarget.isLittleEndian() ||
3003 Subtarget.getSVEVectorSizeInBits() != 128)
3016 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
3018 MI.getMF()->getFunction().needsUnwindTableEntry();
3024 if (Subtarget.isPaired128Slow()) {
3025 switch (
MI.getOpcode()) {
3028 case AArch64::LDURQi:
3029 case AArch64::STURQi:
3030 case AArch64::LDRQui:
3031 case AArch64::STRQui:
3058std::optional<ExtAddrMode>
3063 bool OffsetIsScalable;
3064 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
3065 return std::nullopt;
3068 return std::nullopt;
3083 int64_t OffsetScale = 1;
3088 case AArch64::LDURQi:
3089 case AArch64::STURQi:
3093 case AArch64::LDURDi:
3094 case AArch64::STURDi:
3095 case AArch64::LDURXi:
3096 case AArch64::STURXi:
3100 case AArch64::LDURWi:
3101 case AArch64::LDURSWi:
3102 case AArch64::STURWi:
3106 case AArch64::LDURHi:
3107 case AArch64::STURHi:
3108 case AArch64::LDURHHi:
3109 case AArch64::STURHHi:
3110 case AArch64::LDURSHXi:
3111 case AArch64::LDURSHWi:
3115 case AArch64::LDRBroX:
3116 case AArch64::LDRBBroX:
3117 case AArch64::LDRSBXroX:
3118 case AArch64::LDRSBWroX:
3119 case AArch64::STRBroX:
3120 case AArch64::STRBBroX:
3121 case AArch64::LDURBi:
3122 case AArch64::LDURBBi:
3123 case AArch64::LDURSBXi:
3124 case AArch64::LDURSBWi:
3125 case AArch64::STURBi:
3126 case AArch64::STURBBi:
3127 case AArch64::LDRBui:
3128 case AArch64::LDRBBui:
3129 case AArch64::LDRSBXui:
3130 case AArch64::LDRSBWui:
3131 case AArch64::STRBui:
3132 case AArch64::STRBBui:
3136 case AArch64::LDRQroX:
3137 case AArch64::STRQroX:
3138 case AArch64::LDRQui:
3139 case AArch64::STRQui:
3144 case AArch64::LDRDroX:
3145 case AArch64::STRDroX:
3146 case AArch64::LDRXroX:
3147 case AArch64::STRXroX:
3148 case AArch64::LDRDui:
3149 case AArch64::STRDui:
3150 case AArch64::LDRXui:
3151 case AArch64::STRXui:
3156 case AArch64::LDRWroX:
3157 case AArch64::LDRSWroX:
3158 case AArch64::STRWroX:
3159 case AArch64::LDRWui:
3160 case AArch64::LDRSWui:
3161 case AArch64::STRWui:
3166 case AArch64::LDRHroX:
3167 case AArch64::STRHroX:
3168 case AArch64::LDRHHroX:
3169 case AArch64::STRHHroX:
3170 case AArch64::LDRSHXroX:
3171 case AArch64::LDRSHWroX:
3172 case AArch64::LDRHui:
3173 case AArch64::STRHui:
3174 case AArch64::LDRHHui:
3175 case AArch64::STRHHui:
3176 case AArch64::LDRSHXui:
3177 case AArch64::LDRSHWui:
3185 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
3209 case AArch64::SBFMXri:
3222 AM.
Scale = OffsetScale;
3227 case TargetOpcode::SUBREG_TO_REG: {
3240 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3244 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3245 DefMI.getOperand(1).getReg() != AArch64::WZR ||
3246 DefMI.getOperand(3).getImm() != 0)
3253 AM.
Scale = OffsetScale;
3264 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3265 int64_t NewOffset) ->
bool {
3266 int64_t MinOffset, MaxOffset;
3283 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3284 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3286 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3288 int64_t NewOffset = OldOffset + Disp;
3289 if (!isLegalAddressingMode(NumBytes, NewOffset, 0))
3293 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3303 auto canFoldAddRegIntoAddrMode =
3308 if ((
unsigned)Scale != Scale)
3310 if (!isLegalAddressingMode(NumBytes, 0, Scale))
3322 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3323 Subtarget.isSTRQroSlow();
3332 case AArch64::ADDXri:
3338 return canFoldAddSubImmIntoAddrMode(Disp);
3340 case AArch64::SUBXri:
3346 return canFoldAddSubImmIntoAddrMode(-Disp);
3348 case AArch64::ADDXrs: {
3361 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3363 if (avoidSlowSTRQ(MemI))
3366 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3369 case AArch64::ADDXrr:
3377 if (!OptSize && avoidSlowSTRQ(MemI))
3379 return canFoldAddRegIntoAddrMode(1);
3381 case AArch64::ADDXrx:
3389 if (!OptSize && avoidSlowSTRQ(MemI))
3398 return canFoldAddRegIntoAddrMode(
3413 case AArch64::LDURQi:
3414 case AArch64::LDRQui:
3415 return AArch64::LDRQroX;
3416 case AArch64::STURQi:
3417 case AArch64::STRQui:
3418 return AArch64::STRQroX;
3419 case AArch64::LDURDi:
3420 case AArch64::LDRDui:
3421 return AArch64::LDRDroX;
3422 case AArch64::STURDi:
3423 case AArch64::STRDui:
3424 return AArch64::STRDroX;
3425 case AArch64::LDURXi:
3426 case AArch64::LDRXui:
3427 return AArch64::LDRXroX;
3428 case AArch64::STURXi:
3429 case AArch64::STRXui:
3430 return AArch64::STRXroX;
3431 case AArch64::LDURWi:
3432 case AArch64::LDRWui:
3433 return AArch64::LDRWroX;
3434 case AArch64::LDURSWi:
3435 case AArch64::LDRSWui:
3436 return AArch64::LDRSWroX;
3437 case AArch64::STURWi:
3438 case AArch64::STRWui:
3439 return AArch64::STRWroX;
3440 case AArch64::LDURHi:
3441 case AArch64::LDRHui:
3442 return AArch64::LDRHroX;
3443 case AArch64::STURHi:
3444 case AArch64::STRHui:
3445 return AArch64::STRHroX;
3446 case AArch64::LDURHHi:
3447 case AArch64::LDRHHui:
3448 return AArch64::LDRHHroX;
3449 case AArch64::STURHHi:
3450 case AArch64::STRHHui:
3451 return AArch64::STRHHroX;
3452 case AArch64::LDURSHXi:
3453 case AArch64::LDRSHXui:
3454 return AArch64::LDRSHXroX;
3455 case AArch64::LDURSHWi:
3456 case AArch64::LDRSHWui:
3457 return AArch64::LDRSHWroX;
3458 case AArch64::LDURBi:
3459 case AArch64::LDRBui:
3460 return AArch64::LDRBroX;
3461 case AArch64::LDURBBi:
3462 case AArch64::LDRBBui:
3463 return AArch64::LDRBBroX;
3464 case AArch64::LDURSBXi:
3465 case AArch64::LDRSBXui:
3466 return AArch64::LDRSBXroX;
3467 case AArch64::LDURSBWi:
3468 case AArch64::LDRSBWui:
3469 return AArch64::LDRSBWroX;
3470 case AArch64::STURBi:
3471 case AArch64::STRBui:
3472 return AArch64::STRBroX;
3473 case AArch64::STURBBi:
3474 case AArch64::STRBBui:
3475 return AArch64::STRBBroX;
3487 case AArch64::LDURQi:
3489 return AArch64::LDRQui;
3490 case AArch64::STURQi:
3492 return AArch64::STRQui;
3493 case AArch64::LDURDi:
3495 return AArch64::LDRDui;
3496 case AArch64::STURDi:
3498 return AArch64::STRDui;
3499 case AArch64::LDURXi:
3501 return AArch64::LDRXui;
3502 case AArch64::STURXi:
3504 return AArch64::STRXui;
3505 case AArch64::LDURWi:
3507 return AArch64::LDRWui;
3508 case AArch64::LDURSWi:
3510 return AArch64::LDRSWui;
3511 case AArch64::STURWi:
3513 return AArch64::STRWui;
3514 case AArch64::LDURHi:
3516 return AArch64::LDRHui;
3517 case AArch64::STURHi:
3519 return AArch64::STRHui;
3520 case AArch64::LDURHHi:
3522 return AArch64::LDRHHui;
3523 case AArch64::STURHHi:
3525 return AArch64::STRHHui;
3526 case AArch64::LDURSHXi:
3528 return AArch64::LDRSHXui;
3529 case AArch64::LDURSHWi:
3531 return AArch64::LDRSHWui;
3532 case AArch64::LDURBi:
3534 return AArch64::LDRBui;
3535 case AArch64::LDURBBi:
3537 return AArch64::LDRBBui;
3538 case AArch64::LDURSBXi:
3540 return AArch64::LDRSBXui;
3541 case AArch64::LDURSBWi:
3543 return AArch64::LDRSBWui;
3544 case AArch64::STURBi:
3546 return AArch64::STRBui;
3547 case AArch64::STURBBi:
3549 return AArch64::STRBBui;
3550 case AArch64::LDRQui:
3551 case AArch64::STRQui:
3554 case AArch64::LDRDui:
3555 case AArch64::STRDui:
3556 case AArch64::LDRXui:
3557 case AArch64::STRXui:
3560 case AArch64::LDRWui:
3561 case AArch64::LDRSWui:
3562 case AArch64::STRWui:
3565 case AArch64::LDRHui:
3566 case AArch64::STRHui:
3567 case AArch64::LDRHHui:
3568 case AArch64::STRHHui:
3569 case AArch64::LDRSHXui:
3570 case AArch64::LDRSHWui:
3573 case AArch64::LDRBui:
3574 case AArch64::LDRBBui:
3575 case AArch64::LDRSBXui:
3576 case AArch64::LDRSBWui:
3577 case AArch64::STRBui:
3578 case AArch64::STRBBui:
3592 case AArch64::LDURQi:
3593 case AArch64::STURQi:
3594 case AArch64::LDURDi:
3595 case AArch64::STURDi:
3596 case AArch64::LDURXi:
3597 case AArch64::STURXi:
3598 case AArch64::LDURWi:
3599 case AArch64::LDURSWi:
3600 case AArch64::STURWi:
3601 case AArch64::LDURHi:
3602 case AArch64::STURHi:
3603 case AArch64::LDURHHi:
3604 case AArch64::STURHHi:
3605 case AArch64::LDURSHXi:
3606 case AArch64::LDURSHWi:
3607 case AArch64::LDURBi:
3608 case AArch64::STURBi:
3609 case AArch64::LDURBBi:
3610 case AArch64::STURBBi:
3611 case AArch64::LDURSBWi:
3612 case AArch64::LDURSBXi:
3614 case AArch64::LDRQui:
3615 return AArch64::LDURQi;
3616 case AArch64::STRQui:
3617 return AArch64::STURQi;
3618 case AArch64::LDRDui:
3619 return AArch64::LDURDi;
3620 case AArch64::STRDui:
3621 return AArch64::STURDi;
3622 case AArch64::LDRXui:
3623 return AArch64::LDURXi;
3624 case AArch64::STRXui:
3625 return AArch64::STURXi;
3626 case AArch64::LDRWui:
3627 return AArch64::LDURWi;
3628 case AArch64::LDRSWui:
3629 return AArch64::LDURSWi;
3630 case AArch64::STRWui:
3631 return AArch64::STURWi;
3632 case AArch64::LDRHui:
3633 return AArch64::LDURHi;
3634 case AArch64::STRHui:
3635 return AArch64::STURHi;
3636 case AArch64::LDRHHui:
3637 return AArch64::LDURHHi;
3638 case AArch64::STRHHui:
3639 return AArch64::STURHHi;
3640 case AArch64::LDRSHXui:
3641 return AArch64::LDURSHXi;
3642 case AArch64::LDRSHWui:
3643 return AArch64::LDURSHWi;
3644 case AArch64::LDRBBui:
3645 return AArch64::LDURBBi;
3646 case AArch64::LDRBui:
3647 return AArch64::LDURBi;
3648 case AArch64::STRBBui:
3649 return AArch64::STURBBi;
3650 case AArch64::STRBui:
3651 return AArch64::STURBi;
3652 case AArch64::LDRSBWui:
3653 return AArch64::LDURSBWi;
3654 case AArch64::LDRSBXui:
3655 return AArch64::LDURSBXi;
3668 case AArch64::LDRQroX:
3669 case AArch64::LDURQi:
3670 case AArch64::LDRQui:
3671 return AArch64::LDRQroW;
3672 case AArch64::STRQroX:
3673 case AArch64::STURQi:
3674 case AArch64::STRQui:
3675 return AArch64::STRQroW;
3676 case AArch64::LDRDroX:
3677 case AArch64::LDURDi:
3678 case AArch64::LDRDui:
3679 return AArch64::LDRDroW;
3680 case AArch64::STRDroX:
3681 case AArch64::STURDi:
3682 case AArch64::STRDui:
3683 return AArch64::STRDroW;
3684 case AArch64::LDRXroX:
3685 case AArch64::LDURXi:
3686 case AArch64::LDRXui:
3687 return AArch64::LDRXroW;
3688 case AArch64::STRXroX:
3689 case AArch64::STURXi:
3690 case AArch64::STRXui:
3691 return AArch64::STRXroW;
3692 case AArch64::LDRWroX:
3693 case AArch64::LDURWi:
3694 case AArch64::LDRWui:
3695 return AArch64::LDRWroW;
3696 case AArch64::LDRSWroX:
3697 case AArch64::LDURSWi:
3698 case AArch64::LDRSWui:
3699 return AArch64::LDRSWroW;
3700 case AArch64::STRWroX:
3701 case AArch64::STURWi:
3702 case AArch64::STRWui:
3703 return AArch64::STRWroW;
3704 case AArch64::LDRHroX:
3705 case AArch64::LDURHi:
3706 case AArch64::LDRHui:
3707 return AArch64::LDRHroW;
3708 case AArch64::STRHroX:
3709 case AArch64::STURHi:
3710 case AArch64::STRHui:
3711 return AArch64::STRHroW;
3712 case AArch64::LDRHHroX:
3713 case AArch64::LDURHHi:
3714 case AArch64::LDRHHui:
3715 return AArch64::LDRHHroW;
3716 case AArch64::STRHHroX:
3717 case AArch64::STURHHi:
3718 case AArch64::STRHHui:
3719 return AArch64::STRHHroW;
3720 case AArch64::LDRSHXroX:
3721 case AArch64::LDURSHXi:
3722 case AArch64::LDRSHXui:
3723 return AArch64::LDRSHXroW;
3724 case AArch64::LDRSHWroX:
3725 case AArch64::LDURSHWi:
3726 case AArch64::LDRSHWui:
3727 return AArch64::LDRSHWroW;
3728 case AArch64::LDRBroX:
3729 case AArch64::LDURBi:
3730 case AArch64::LDRBui:
3731 return AArch64::LDRBroW;
3732 case AArch64::LDRBBroX:
3733 case AArch64::LDURBBi:
3734 case AArch64::LDRBBui:
3735 return AArch64::LDRBBroW;
3736 case AArch64::LDRSBXroX:
3737 case AArch64::LDURSBXi:
3738 case AArch64::LDRSBXui:
3739 return AArch64::LDRSBXroW;
3740 case AArch64::LDRSBWroX:
3741 case AArch64::LDURSBWi:
3742 case AArch64::LDRSBWui:
3743 return AArch64::LDRSBWroW;
3744 case AArch64::STRBroX:
3745 case AArch64::STURBi:
3746 case AArch64::STRBui:
3747 return AArch64::STRBroW;
3748 case AArch64::STRBBroX:
3749 case AArch64::STURBBi:
3750 case AArch64::STRBBui:
3751 return AArch64::STRBBroW;
3766 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3776 return B.getInstr();
3780 "Addressing mode not supported for folding");
3797 return B.getInstr();
3804 "Address offset can be a register or an immediate, but not both");
3806 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3811 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3825 return B.getInstr();
3829 "Function must not be called with an addressing mode it can't handle");
3838 case AArch64::LD1Fourv16b_POST:
3839 case AArch64::LD1Fourv1d_POST:
3840 case AArch64::LD1Fourv2d_POST:
3841 case AArch64::LD1Fourv2s_POST:
3842 case AArch64::LD1Fourv4h_POST:
3843 case AArch64::LD1Fourv4s_POST:
3844 case AArch64::LD1Fourv8b_POST:
3845 case AArch64::LD1Fourv8h_POST:
3846 case AArch64::LD1Onev16b_POST:
3847 case AArch64::LD1Onev1d_POST:
3848 case AArch64::LD1Onev2d_POST:
3849 case AArch64::LD1Onev2s_POST:
3850 case AArch64::LD1Onev4h_POST:
3851 case AArch64::LD1Onev4s_POST:
3852 case AArch64::LD1Onev8b_POST:
3853 case AArch64::LD1Onev8h_POST:
3854 case AArch64::LD1Rv16b_POST:
3855 case AArch64::LD1Rv1d_POST:
3856 case AArch64::LD1Rv2d_POST:
3857 case AArch64::LD1Rv2s_POST:
3858 case AArch64::LD1Rv4h_POST:
3859 case AArch64::LD1Rv4s_POST:
3860 case AArch64::LD1Rv8b_POST:
3861 case AArch64::LD1Rv8h_POST:
3862 case AArch64::LD1Threev16b_POST:
3863 case AArch64::LD1Threev1d_POST:
3864 case AArch64::LD1Threev2d_POST:
3865 case AArch64::LD1Threev2s_POST:
3866 case AArch64::LD1Threev4h_POST:
3867 case AArch64::LD1Threev4s_POST:
3868 case AArch64::LD1Threev8b_POST:
3869 case AArch64::LD1Threev8h_POST:
3870 case AArch64::LD1Twov16b_POST:
3871 case AArch64::LD1Twov1d_POST:
3872 case AArch64::LD1Twov2d_POST:
3873 case AArch64::LD1Twov2s_POST:
3874 case AArch64::LD1Twov4h_POST:
3875 case AArch64::LD1Twov4s_POST:
3876 case AArch64::LD1Twov8b_POST:
3877 case AArch64::LD1Twov8h_POST:
3878 case AArch64::LD1i16_POST:
3879 case AArch64::LD1i32_POST:
3880 case AArch64::LD1i64_POST:
3881 case AArch64::LD1i8_POST:
3882 case AArch64::LD2Rv16b_POST:
3883 case AArch64::LD2Rv1d_POST:
3884 case AArch64::LD2Rv2d_POST:
3885 case AArch64::LD2Rv2s_POST:
3886 case AArch64::LD2Rv4h_POST:
3887 case AArch64::LD2Rv4s_POST:
3888 case AArch64::LD2Rv8b_POST:
3889 case AArch64::LD2Rv8h_POST:
3890 case AArch64::LD2Twov16b_POST:
3891 case AArch64::LD2Twov2d_POST:
3892 case AArch64::LD2Twov2s_POST:
3893 case AArch64::LD2Twov4h_POST:
3894 case AArch64::LD2Twov4s_POST:
3895 case AArch64::LD2Twov8b_POST:
3896 case AArch64::LD2Twov8h_POST:
3897 case AArch64::LD2i16_POST:
3898 case AArch64::LD2i32_POST:
3899 case AArch64::LD2i64_POST:
3900 case AArch64::LD2i8_POST:
3901 case AArch64::LD3Rv16b_POST:
3902 case AArch64::LD3Rv1d_POST:
3903 case AArch64::LD3Rv2d_POST:
3904 case AArch64::LD3Rv2s_POST:
3905 case AArch64::LD3Rv4h_POST:
3906 case AArch64::LD3Rv4s_POST:
3907 case AArch64::LD3Rv8b_POST:
3908 case AArch64::LD3Rv8h_POST:
3909 case AArch64::LD3Threev16b_POST:
3910 case AArch64::LD3Threev2d_POST:
3911 case AArch64::LD3Threev2s_POST:
3912 case AArch64::LD3Threev4h_POST:
3913 case AArch64::LD3Threev4s_POST:
3914 case AArch64::LD3Threev8b_POST:
3915 case AArch64::LD3Threev8h_POST:
3916 case AArch64::LD3i16_POST:
3917 case AArch64::LD3i32_POST:
3918 case AArch64::LD3i64_POST:
3919 case AArch64::LD3i8_POST:
3920 case AArch64::LD4Fourv16b_POST:
3921 case AArch64::LD4Fourv2d_POST:
3922 case AArch64::LD4Fourv2s_POST:
3923 case AArch64::LD4Fourv4h_POST:
3924 case AArch64::LD4Fourv4s_POST:
3925 case AArch64::LD4Fourv8b_POST:
3926 case AArch64::LD4Fourv8h_POST:
3927 case AArch64::LD4Rv16b_POST:
3928 case AArch64::LD4Rv1d_POST:
3929 case AArch64::LD4Rv2d_POST:
3930 case AArch64::LD4Rv2s_POST:
3931 case AArch64::LD4Rv4h_POST:
3932 case AArch64::LD4Rv4s_POST:
3933 case AArch64::LD4Rv8b_POST:
3934 case AArch64::LD4Rv8h_POST:
3935 case AArch64::LD4i16_POST:
3936 case AArch64::LD4i32_POST:
3937 case AArch64::LD4i64_POST:
3938 case AArch64::LD4i8_POST:
3939 case AArch64::LDAPRWpost:
3940 case AArch64::LDAPRXpost:
3941 case AArch64::LDIAPPWpost:
3942 case AArch64::LDIAPPXpost:
3943 case AArch64::LDPDpost:
3944 case AArch64::LDPQpost:
3945 case AArch64::LDPSWpost:
3946 case AArch64::LDPSpost:
3947 case AArch64::LDPWpost:
3948 case AArch64::LDPXpost:
3949 case AArch64::LDRBBpost:
3950 case AArch64::LDRBpost:
3951 case AArch64::LDRDpost:
3952 case AArch64::LDRHHpost:
3953 case AArch64::LDRHpost:
3954 case AArch64::LDRQpost:
3955 case AArch64::LDRSBWpost:
3956 case AArch64::LDRSBXpost:
3957 case AArch64::LDRSHWpost:
3958 case AArch64::LDRSHXpost:
3959 case AArch64::LDRSWpost:
3960 case AArch64::LDRSpost:
3961 case AArch64::LDRWpost:
3962 case AArch64::LDRXpost:
3963 case AArch64::ST1Fourv16b_POST:
3964 case AArch64::ST1Fourv1d_POST:
3965 case AArch64::ST1Fourv2d_POST:
3966 case AArch64::ST1Fourv2s_POST:
3967 case AArch64::ST1Fourv4h_POST:
3968 case AArch64::ST1Fourv4s_POST:
3969 case AArch64::ST1Fourv8b_POST:
3970 case AArch64::ST1Fourv8h_POST:
3971 case AArch64::ST1Onev16b_POST:
3972 case AArch64::ST1Onev1d_POST:
3973 case AArch64::ST1Onev2d_POST:
3974 case AArch64::ST1Onev2s_POST:
3975 case AArch64::ST1Onev4h_POST:
3976 case AArch64::ST1Onev4s_POST:
3977 case AArch64::ST1Onev8b_POST:
3978 case AArch64::ST1Onev8h_POST:
3979 case AArch64::ST1Threev16b_POST:
3980 case AArch64::ST1Threev1d_POST:
3981 case AArch64::ST1Threev2d_POST:
3982 case AArch64::ST1Threev2s_POST:
3983 case AArch64::ST1Threev4h_POST:
3984 case AArch64::ST1Threev4s_POST:
3985 case AArch64::ST1Threev8b_POST:
3986 case AArch64::ST1Threev8h_POST:
3987 case AArch64::ST1Twov16b_POST:
3988 case AArch64::ST1Twov1d_POST:
3989 case AArch64::ST1Twov2d_POST:
3990 case AArch64::ST1Twov2s_POST:
3991 case AArch64::ST1Twov4h_POST:
3992 case AArch64::ST1Twov4s_POST:
3993 case AArch64::ST1Twov8b_POST:
3994 case AArch64::ST1Twov8h_POST:
3995 case AArch64::ST1i16_POST:
3996 case AArch64::ST1i32_POST:
3997 case AArch64::ST1i64_POST:
3998 case AArch64::ST1i8_POST:
3999 case AArch64::ST2GPostIndex:
4000 case AArch64::ST2Twov16b_POST:
4001 case AArch64::ST2Twov2d_POST:
4002 case AArch64::ST2Twov2s_POST:
4003 case AArch64::ST2Twov4h_POST:
4004 case AArch64::ST2Twov4s_POST:
4005 case AArch64::ST2Twov8b_POST:
4006 case AArch64::ST2Twov8h_POST:
4007 case AArch64::ST2i16_POST:
4008 case AArch64::ST2i32_POST:
4009 case AArch64::ST2i64_POST:
4010 case AArch64::ST2i8_POST:
4011 case AArch64::ST3Threev16b_POST:
4012 case AArch64::ST3Threev2d_POST:
4013 case AArch64::ST3Threev2s_POST:
4014 case AArch64::ST3Threev4h_POST:
4015 case AArch64::ST3Threev4s_POST:
4016 case AArch64::ST3Threev8b_POST:
4017 case AArch64::ST3Threev8h_POST:
4018 case AArch64::ST3i16_POST:
4019 case AArch64::ST3i32_POST:
4020 case AArch64::ST3i64_POST:
4021 case AArch64::ST3i8_POST:
4022 case AArch64::ST4Fourv16b_POST:
4023 case AArch64::ST4Fourv2d_POST:
4024 case AArch64::ST4Fourv2s_POST:
4025 case AArch64::ST4Fourv4h_POST:
4026 case AArch64::ST4Fourv4s_POST:
4027 case AArch64::ST4Fourv8b_POST:
4028 case AArch64::ST4Fourv8h_POST:
4029 case AArch64::ST4i16_POST:
4030 case AArch64::ST4i32_POST:
4031 case AArch64::ST4i64_POST:
4032 case AArch64::ST4i8_POST:
4033 case AArch64::STGPostIndex:
4034 case AArch64::STGPpost:
4035 case AArch64::STPDpost:
4036 case AArch64::STPQpost:
4037 case AArch64::STPSpost:
4038 case AArch64::STPWpost:
4039 case AArch64::STPXpost:
4040 case AArch64::STRBBpost:
4041 case AArch64::STRBpost:
4042 case AArch64::STRDpost:
4043 case AArch64::STRHHpost:
4044 case AArch64::STRHpost:
4045 case AArch64::STRQpost:
4046 case AArch64::STRSpost:
4047 case AArch64::STRWpost:
4048 case AArch64::STRXpost:
4049 case AArch64::STZ2GPostIndex:
4050 case AArch64::STZGPostIndex:
4057 bool &OffsetIsScalable,
TypeSize &Width,
4078 int64_t Dummy1, Dummy2;
4100 return BaseOp->
isReg() || BaseOp->
isFI();
4107 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
4112 TypeSize &Width, int64_t &MinOffset,
4113 int64_t &MaxOffset) {
4119 MinOffset = MaxOffset = 0;
4122 case AArch64::LDRQui:
4123 case AArch64::STRQui:
4129 case AArch64::LDRXui:
4130 case AArch64::LDRDui:
4131 case AArch64::STRXui:
4132 case AArch64::STRDui:
4133 case AArch64::PRFMui:
4139 case AArch64::LDRWui:
4140 case AArch64::LDRSui:
4141 case AArch64::LDRSWui:
4142 case AArch64::STRWui:
4143 case AArch64::STRSui:
4149 case AArch64::LDRHui:
4150 case AArch64::LDRHHui:
4151 case AArch64::LDRSHWui:
4152 case AArch64::LDRSHXui:
4153 case AArch64::STRHui:
4154 case AArch64::STRHHui:
4160 case AArch64::LDRBui:
4161 case AArch64::LDRBBui:
4162 case AArch64::LDRSBWui:
4163 case AArch64::LDRSBXui:
4164 case AArch64::STRBui:
4165 case AArch64::STRBBui:
4172 case AArch64::STRQpre:
4173 case AArch64::LDRQpost:
4179 case AArch64::LDRDpost:
4180 case AArch64::LDRDpre:
4181 case AArch64::LDRXpost:
4182 case AArch64::LDRXpre:
4183 case AArch64::STRDpost:
4184 case AArch64::STRDpre:
4185 case AArch64::STRXpost:
4186 case AArch64::STRXpre:
4192 case AArch64::STRWpost:
4193 case AArch64::STRWpre:
4194 case AArch64::LDRWpost:
4195 case AArch64::LDRWpre:
4196 case AArch64::STRSpost:
4197 case AArch64::STRSpre:
4198 case AArch64::LDRSpost:
4199 case AArch64::LDRSpre:
4205 case AArch64::LDRHpost:
4206 case AArch64::LDRHpre:
4207 case AArch64::STRHpost:
4208 case AArch64::STRHpre:
4209 case AArch64::LDRHHpost:
4210 case AArch64::LDRHHpre:
4211 case AArch64::STRHHpost:
4212 case AArch64::STRHHpre:
4218 case AArch64::LDRBpost:
4219 case AArch64::LDRBpre:
4220 case AArch64::STRBpost:
4221 case AArch64::STRBpre:
4222 case AArch64::LDRBBpost:
4223 case AArch64::LDRBBpre:
4224 case AArch64::STRBBpost:
4225 case AArch64::STRBBpre:
4232 case AArch64::LDURQi:
4233 case AArch64::STURQi:
4239 case AArch64::LDURXi:
4240 case AArch64::LDURDi:
4241 case AArch64::LDAPURXi:
4242 case AArch64::STURXi:
4243 case AArch64::STURDi:
4244 case AArch64::STLURXi:
4245 case AArch64::PRFUMi:
4251 case AArch64::LDURWi:
4252 case AArch64::LDURSi:
4253 case AArch64::LDURSWi:
4254 case AArch64::LDAPURi:
4255 case AArch64::LDAPURSWi:
4256 case AArch64::STURWi:
4257 case AArch64::STURSi:
4258 case AArch64::STLURWi:
4264 case AArch64::LDURHi:
4265 case AArch64::LDURHHi:
4266 case AArch64::LDURSHXi:
4267 case AArch64::LDURSHWi:
4268 case AArch64::LDAPURHi:
4269 case AArch64::LDAPURSHWi:
4270 case AArch64::LDAPURSHXi:
4271 case AArch64::STURHi:
4272 case AArch64::STURHHi:
4273 case AArch64::STLURHi:
4279 case AArch64::LDURBi:
4280 case AArch64::LDURBBi:
4281 case AArch64::LDURSBXi:
4282 case AArch64::LDURSBWi:
4283 case AArch64::LDAPURBi:
4284 case AArch64::LDAPURSBWi:
4285 case AArch64::LDAPURSBXi:
4286 case AArch64::STURBi:
4287 case AArch64::STURBBi:
4288 case AArch64::STLURBi:
4295 case AArch64::LDPQi:
4296 case AArch64::LDNPQi:
4297 case AArch64::STPQi:
4298 case AArch64::STNPQi:
4299 case AArch64::LDPQpost:
4300 case AArch64::LDPQpre:
4301 case AArch64::STPQpost:
4302 case AArch64::STPQpre:
4308 case AArch64::LDPXi:
4309 case AArch64::LDPDi:
4310 case AArch64::LDNPXi:
4311 case AArch64::LDNPDi:
4312 case AArch64::STPXi:
4313 case AArch64::STPDi:
4314 case AArch64::STNPXi:
4315 case AArch64::STNPDi:
4316 case AArch64::LDPDpost:
4317 case AArch64::LDPDpre:
4318 case AArch64::LDPXpost:
4319 case AArch64::LDPXpre:
4320 case AArch64::STPDpost:
4321 case AArch64::STPDpre:
4322 case AArch64::STPXpost:
4323 case AArch64::STPXpre:
4329 case AArch64::LDPWi:
4330 case AArch64::LDPSi:
4331 case AArch64::LDNPWi:
4332 case AArch64::LDNPSi:
4333 case AArch64::STPWi:
4334 case AArch64::STPSi:
4335 case AArch64::STNPWi:
4336 case AArch64::STNPSi:
4337 case AArch64::LDPSpost:
4338 case AArch64::LDPSpre:
4339 case AArch64::LDPWpost:
4340 case AArch64::LDPWpre:
4341 case AArch64::STPSpost:
4342 case AArch64::STPSpre:
4343 case AArch64::STPWpost:
4344 case AArch64::STPWpre:
4350 case AArch64::StoreSwiftAsyncContext:
4363 case AArch64::TAGPstack:
4373 case AArch64::STGPreIndex:
4374 case AArch64::STGPostIndex:
4375 case AArch64::STZGi:
4376 case AArch64::STZGPreIndex:
4377 case AArch64::STZGPostIndex:
4384 case AArch64::STR_ZZZZXI:
4385 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
4386 case AArch64::LDR_ZZZZXI:
4387 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
4393 case AArch64::STR_ZZZXI:
4394 case AArch64::LDR_ZZZXI:
4400 case AArch64::STR_ZZXI:
4401 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
4402 case AArch64::LDR_ZZXI:
4403 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
4409 case AArch64::LDR_PXI:
4410 case AArch64::STR_PXI:
4416 case AArch64::LDR_PPXI:
4417 case AArch64::STR_PPXI:
4423 case AArch64::LDR_ZXI:
4424 case AArch64::STR_ZXI:
4430 case AArch64::LD1B_IMM:
4431 case AArch64::LD1H_IMM:
4432 case AArch64::LD1W_IMM:
4433 case AArch64::LD1D_IMM:
4434 case AArch64::LDNT1B_ZRI:
4435 case AArch64::LDNT1H_ZRI:
4436 case AArch64::LDNT1W_ZRI:
4437 case AArch64::LDNT1D_ZRI:
4438 case AArch64::ST1B_IMM:
4439 case AArch64::ST1H_IMM:
4440 case AArch64::ST1W_IMM:
4441 case AArch64::ST1D_IMM:
4442 case AArch64::STNT1B_ZRI:
4443 case AArch64::STNT1H_ZRI:
4444 case AArch64::STNT1W_ZRI:
4445 case AArch64::STNT1D_ZRI:
4446 case AArch64::LDNF1B_IMM:
4447 case AArch64::LDNF1H_IMM:
4448 case AArch64::LDNF1W_IMM:
4449 case AArch64::LDNF1D_IMM:
4457 case AArch64::LD2B_IMM:
4458 case AArch64::LD2H_IMM:
4459 case AArch64::LD2W_IMM:
4460 case AArch64::LD2D_IMM:
4461 case AArch64::ST2B_IMM:
4462 case AArch64::ST2H_IMM:
4463 case AArch64::ST2W_IMM:
4464 case AArch64::ST2D_IMM:
4470 case AArch64::LD3B_IMM:
4471 case AArch64::LD3H_IMM:
4472 case AArch64::LD3W_IMM:
4473 case AArch64::LD3D_IMM:
4474 case AArch64::ST3B_IMM:
4475 case AArch64::ST3H_IMM:
4476 case AArch64::ST3W_IMM:
4477 case AArch64::ST3D_IMM:
4483 case AArch64::LD4B_IMM:
4484 case AArch64::LD4H_IMM:
4485 case AArch64::LD4W_IMM:
4486 case AArch64::LD4D_IMM:
4487 case AArch64::ST4B_IMM:
4488 case AArch64::ST4H_IMM:
4489 case AArch64::ST4W_IMM:
4490 case AArch64::ST4D_IMM:
4496 case AArch64::LD1B_H_IMM:
4497 case AArch64::LD1SB_H_IMM:
4498 case AArch64::LD1H_S_IMM:
4499 case AArch64::LD1SH_S_IMM:
4500 case AArch64::LD1W_D_IMM:
4501 case AArch64::LD1SW_D_IMM:
4502 case AArch64::ST1B_H_IMM:
4503 case AArch64::ST1H_S_IMM:
4504 case AArch64::ST1W_D_IMM:
4505 case AArch64::LDNF1B_H_IMM:
4506 case AArch64::LDNF1SB_H_IMM:
4507 case AArch64::LDNF1H_S_IMM:
4508 case AArch64::LDNF1SH_S_IMM:
4509 case AArch64::LDNF1W_D_IMM:
4510 case AArch64::LDNF1SW_D_IMM:
4518 case AArch64::LD1B_S_IMM:
4519 case AArch64::LD1SB_S_IMM:
4520 case AArch64::LD1H_D_IMM:
4521 case AArch64::LD1SH_D_IMM:
4522 case AArch64::ST1B_S_IMM:
4523 case AArch64::ST1H_D_IMM:
4524 case AArch64::LDNF1B_S_IMM:
4525 case AArch64::LDNF1SB_S_IMM:
4526 case AArch64::LDNF1H_D_IMM:
4527 case AArch64::LDNF1SH_D_IMM:
4535 case AArch64::LD1B_D_IMM:
4536 case AArch64::LD1SB_D_IMM:
4537 case AArch64::ST1B_D_IMM:
4538 case AArch64::LDNF1B_D_IMM:
4539 case AArch64::LDNF1SB_D_IMM:
4547 case AArch64::ST2Gi:
4548 case AArch64::ST2GPreIndex:
4549 case AArch64::ST2GPostIndex:
4550 case AArch64::STZ2Gi:
4551 case AArch64::STZ2GPreIndex:
4552 case AArch64::STZ2GPostIndex:
4558 case AArch64::STGPi:
4559 case AArch64::STGPpost:
4560 case AArch64::STGPpre:
4566 case AArch64::LD1RB_IMM:
4567 case AArch64::LD1RB_H_IMM:
4568 case AArch64::LD1RB_S_IMM:
4569 case AArch64::LD1RB_D_IMM:
4570 case AArch64::LD1RSB_H_IMM:
4571 case AArch64::LD1RSB_S_IMM:
4572 case AArch64::LD1RSB_D_IMM:
4578 case AArch64::LD1RH_IMM:
4579 case AArch64::LD1RH_S_IMM:
4580 case AArch64::LD1RH_D_IMM:
4581 case AArch64::LD1RSH_S_IMM:
4582 case AArch64::LD1RSH_D_IMM:
4588 case AArch64::LD1RW_IMM:
4589 case AArch64::LD1RW_D_IMM:
4590 case AArch64::LD1RSW_IMM:
4596 case AArch64::LD1RD_IMM:
4612 case AArch64::LDRBBui:
4613 case AArch64::LDURBBi:
4614 case AArch64::LDRSBWui:
4615 case AArch64::LDURSBWi:
4616 case AArch64::STRBBui:
4617 case AArch64::STURBBi:
4619 case AArch64::LDRHHui:
4620 case AArch64::LDURHHi:
4621 case AArch64::LDRSHWui:
4622 case AArch64::LDURSHWi:
4623 case AArch64::STRHHui:
4624 case AArch64::STURHHi:
4626 case AArch64::LDRSui:
4627 case AArch64::LDURSi:
4628 case AArch64::LDRSpre:
4629 case AArch64::LDRSWui:
4630 case AArch64::LDURSWi:
4631 case AArch64::LDRSWpre:
4632 case AArch64::LDRWpre:
4633 case AArch64::LDRWui:
4634 case AArch64::LDURWi:
4635 case AArch64::STRSui:
4636 case AArch64::STURSi:
4637 case AArch64::STRSpre:
4638 case AArch64::STRWui:
4639 case AArch64::STURWi:
4640 case AArch64::STRWpre:
4641 case AArch64::LDPSi:
4642 case AArch64::LDPSWi:
4643 case AArch64::LDPWi:
4644 case AArch64::STPSi:
4645 case AArch64::STPWi:
4647 case AArch64::LDRDui:
4648 case AArch64::LDURDi:
4649 case AArch64::LDRDpre:
4650 case AArch64::LDRXui:
4651 case AArch64::LDURXi:
4652 case AArch64::LDRXpre:
4653 case AArch64::STRDui:
4654 case AArch64::STURDi:
4655 case AArch64::STRDpre:
4656 case AArch64::STRXui:
4657 case AArch64::STURXi:
4658 case AArch64::STRXpre:
4659 case AArch64::LDPDi:
4660 case AArch64::LDPXi:
4661 case AArch64::STPDi:
4662 case AArch64::STPXi:
4664 case AArch64::LDRQui:
4665 case AArch64::LDURQi:
4666 case AArch64::STRQui:
4667 case AArch64::STURQi:
4668 case AArch64::STRQpre:
4669 case AArch64::LDPQi:
4670 case AArch64::LDRQpre:
4671 case AArch64::STPQi:
4673 case AArch64::STZGi:
4674 case AArch64::ST2Gi:
4675 case AArch64::STZ2Gi:
4676 case AArch64::STGPi:
4682 switch (
MI.getOpcode()) {
4685 case AArch64::LDRWpre:
4686 case AArch64::LDRXpre:
4687 case AArch64::LDRSWpre:
4688 case AArch64::LDRSpre:
4689 case AArch64::LDRDpre:
4690 case AArch64::LDRQpre:
4696 switch (
MI.getOpcode()) {
4699 case AArch64::STRWpre:
4700 case AArch64::STRXpre:
4701 case AArch64::STRSpre:
4702 case AArch64::STRDpre:
4703 case AArch64::STRQpre:
4713 switch (
MI.getOpcode()) {
4716 case AArch64::LDPSi:
4717 case AArch64::LDPSWi:
4718 case AArch64::LDPDi:
4719 case AArch64::LDPQi:
4720 case AArch64::LDPWi:
4721 case AArch64::LDPXi:
4722 case AArch64::STPSi:
4723 case AArch64::STPDi:
4724 case AArch64::STPQi:
4725 case AArch64::STPWi:
4726 case AArch64::STPXi:
4727 case AArch64::STGPi:
4733 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4737 return MI.getOperand(Idx);
4742 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4746 return MI.getOperand(Idx);
4751 switch (
MI.getOpcode()) {
4754 case AArch64::LDRBroX:
4755 case AArch64::LDRBBroX:
4756 case AArch64::LDRSBXroX:
4757 case AArch64::LDRSBWroX:
4758 case AArch64::LDRHroX:
4759 case AArch64::LDRHHroX:
4760 case AArch64::LDRSHXroX:
4761 case AArch64::LDRSHWroX:
4762 case AArch64::LDRWroX:
4763 case AArch64::LDRSroX:
4764 case AArch64::LDRSWroX:
4765 case AArch64::LDRDroX:
4766 case AArch64::LDRXroX:
4767 case AArch64::LDRQroX:
4768 return MI.getOperand(4);
4774 if (
MI.getParent() ==
nullptr)
4784 auto Reg =
Op.getReg();
4785 if (Reg.isPhysical())
4786 return AArch64::FPR16RegClass.contains(Reg);
4788 return TRC == &AArch64::FPR16RegClass ||
4789 TRC == &AArch64::FPR16_loRegClass;
4798 auto Reg =
Op.getReg();
4799 if (Reg.isPhysical())
4800 return AArch64::FPR128RegClass.contains(Reg);
4802 return TRC == &AArch64::FPR128RegClass ||
4803 TRC == &AArch64::FPR128_loRegClass;
4809 switch (
MI.getOpcode()) {
4812 case AArch64::PACIASP:
4813 case AArch64::PACIBSP:
4816 case AArch64::PAUTH_PROLOGUE:
4819 case AArch64::HINT: {
4820 unsigned Imm =
MI.getOperand(0).getImm();
4822 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4825 if (Imm == 25 || Imm == 27)
4837 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4838 return AArch64::FPR128RegClass.contains(Reg) ||
4839 AArch64::FPR64RegClass.contains(Reg) ||
4840 AArch64::FPR32RegClass.contains(Reg) ||
4841 AArch64::FPR16RegClass.contains(Reg) ||
4842 AArch64::FPR8RegClass.contains(Reg);
4849 auto Reg =
Op.getReg();
4850 if (Reg.isPhysical())
4854 return TRC == &AArch64::FPR128RegClass ||
4855 TRC == &AArch64::FPR128_loRegClass ||
4856 TRC == &AArch64::FPR64RegClass ||
4857 TRC == &AArch64::FPR64_loRegClass ||
4858 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4859 TRC == &AArch64::FPR8RegClass;
4881 if (FirstOpc == SecondOpc)
4887 case AArch64::STRSui:
4888 case AArch64::STURSi:
4889 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4890 case AArch64::STRDui:
4891 case AArch64::STURDi:
4892 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4893 case AArch64::STRQui:
4894 case AArch64::STURQi:
4895 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4896 case AArch64::STRWui:
4897 case AArch64::STURWi:
4898 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4899 case AArch64::STRXui:
4900 case AArch64::STURXi:
4901 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4902 case AArch64::LDRSui:
4903 case AArch64::LDURSi:
4904 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4905 case AArch64::LDRDui:
4906 case AArch64::LDURDi:
4907 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4908 case AArch64::LDRQui:
4909 case AArch64::LDURQi:
4910 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4911 case AArch64::LDRWui:
4912 case AArch64::LDURWi:
4913 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4914 case AArch64::LDRSWui:
4915 case AArch64::LDURSWi:
4916 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4917 case AArch64::LDRXui:
4918 case AArch64::LDURXi:
4919 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4926 int64_t Offset1,
unsigned Opcode1,
int FI2,
4927 int64_t Offset2,
unsigned Opcode2) {
4933 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4936 if (ObjectOffset1 % Scale1 != 0)
4938 ObjectOffset1 /= Scale1;
4940 if (ObjectOffset2 % Scale2 != 0)
4942 ObjectOffset2 /= Scale2;
4943 ObjectOffset1 += Offset1;
4944 ObjectOffset2 += Offset2;
4945 return ObjectOffset1 + 1 == ObjectOffset2;
4957 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4958 unsigned NumBytes)
const {
4968 "Only base registers and frame indices are supported.");
4975 if (ClusterSize > 2)
4982 unsigned FirstOpc = FirstLdSt.
getOpcode();
4983 unsigned SecondOpc = SecondLdSt.
getOpcode();
5003 if (Offset1 > 63 || Offset1 < -64)
5008 if (BaseOp1.
isFI()) {
5010 "Caller should have ordered offsets.");
5015 BaseOp2.
getIndex(), Offset2, SecondOpc);
5018 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
5020 return Offset1 + 1 == Offset2;
5030 if (
Reg.isPhysical())
5039 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
5048 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
5050 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
5051 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
5052 unsigned NumRegs = Indices.
size();
5054 int SubReg = 0, End = NumRegs, Incr = 1;
5073 unsigned Opcode,
unsigned ZeroReg,
5076 unsigned NumRegs = Indices.
size();
5079 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
5080 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
5081 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5082 "GPR reg sequences should not be able to overlap");
5099 bool RenamableSrc)
const {
5100 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
5101 AArch64::GPR32spRegClass.
contains(SrcReg)) {
5102 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5104 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5105 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5107 MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
5108 &AArch64::GPR64spRegClass);
5109 MCRegister SrcRegX = RI.getMatchingSuperReg(SrcReg, AArch64::sub_32,
5110 &AArch64::GPR64spRegClass);
5126 }
else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5127 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5129 MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
5130 &AArch64::GPR64spRegClass);
5131 assert(DestRegX.
isValid() &&
"Destination super-reg not valid");
5132 MCRegister SrcRegX = RI.getMatchingSuperReg(SrcReg, AArch64::sub_32,
5133 &AArch64::GPR64spRegClass);
5153 if (AArch64::GPR32spRegClass.
contains(DestReg) && SrcReg == AArch64::WZR) {
5154 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5155 !Subtarget.hasZeroCycleZeroingGPR32()) {
5156 MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
5157 &AArch64::GPR64spRegClass);
5158 assert(DestRegX.
isValid() &&
"Destination super-reg not valid");
5162 }
else if (Subtarget.hasZeroCycleZeroingGPR32()) {
5174 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5175 AArch64::GPR64spRegClass.
contains(SrcReg)) {
5176 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5192 if (AArch64::GPR64spRegClass.
contains(DestReg) && SrcReg == AArch64::XZR) {
5193 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5206 if (AArch64::PPRRegClass.
contains(DestReg) &&
5207 AArch64::PPRRegClass.
contains(SrcReg)) {
5208 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5209 "Unexpected SVE register.");
5219 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
5220 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
5221 if (DestIsPNR || SrcIsPNR) {
5223 return (R - AArch64::PN0) + AArch64::P0;
5228 if (PPRSrcReg != PPRDestReg) {
5240 if (AArch64::ZPRRegClass.
contains(DestReg) &&
5241 AArch64::ZPRRegClass.
contains(SrcReg)) {
5242 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5243 "Unexpected SVE register.");
5251 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
5252 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
5253 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
5254 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
5255 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5256 "Unexpected SVE register.");
5257 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5264 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
5265 AArch64::ZPR3RegClass.
contains(SrcReg)) {
5266 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5267 "Unexpected SVE register.");
5268 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5276 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
5277 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
5278 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
5279 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5280 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5281 "Unexpected SVE register.");
5282 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5283 AArch64::zsub2, AArch64::zsub3};
5290 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5291 AArch64::DDDDRegClass.
contains(SrcReg)) {
5292 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5293 AArch64::dsub2, AArch64::dsub3};
5300 if (AArch64::DDDRegClass.
contains(DestReg) &&
5301 AArch64::DDDRegClass.
contains(SrcReg)) {
5302 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5310 if (AArch64::DDRegClass.
contains(DestReg) &&
5311 AArch64::DDRegClass.
contains(SrcReg)) {
5312 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5319 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5320 AArch64::QQQQRegClass.
contains(SrcReg)) {
5321 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5322 AArch64::qsub2, AArch64::qsub3};
5329 if (AArch64::QQQRegClass.
contains(DestReg) &&
5330 AArch64::QQQRegClass.
contains(SrcReg)) {
5331 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5339 if (AArch64::QQRegClass.
contains(DestReg) &&
5340 AArch64::QQRegClass.
contains(SrcReg)) {
5341 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5347 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5348 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5349 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5351 AArch64::XZR, Indices);
5355 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5356 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5357 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5359 AArch64::WZR, Indices);
5363 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5364 AArch64::FPR128RegClass.
contains(SrcReg)) {
5365 if (Subtarget.isSVEorStreamingSVEAvailable() &&
5366 !Subtarget.isNeonAvailable())
5369 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5370 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5371 else if (Subtarget.isNeonAvailable())
5390 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5391 AArch64::FPR64RegClass.
contains(SrcReg)) {
5392 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5393 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5394 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5395 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
5396 &AArch64::FPR128RegClass);
5397 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
5398 &AArch64::FPR128RegClass);
5414 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5415 AArch64::FPR32RegClass.
contains(SrcReg)) {
5416 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5417 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5418 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5419 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
5420 &AArch64::FPR128RegClass);
5421 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
5422 &AArch64::FPR128RegClass);
5431 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5432 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5433 MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
5434 &AArch64::FPR64RegClass);
5435 MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
5436 &AArch64::FPR64RegClass);
5451 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5452 AArch64::FPR16RegClass.
contains(SrcReg)) {
5453 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5454 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5455 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) {
5456 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5457 &AArch64::FPR128RegClass);
5458 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5459 &AArch64::FPR128RegClass);
5468 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5469 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5470 MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5471 &AArch64::FPR64RegClass);
5472 MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5473 &AArch64::FPR64RegClass);
5482 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
5483 &AArch64::FPR32RegClass);
5484 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
5485 &AArch64::FPR32RegClass);
5492 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5493 AArch64::FPR8RegClass.
contains(SrcReg)) {
5494 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
5495 !Subtarget.hasZeroCycleRegMoveFPR64() &&
5496 !Subtarget.hasZeroCycleRegMoveFPR64() && Subtarget.isNeonAvailable()) {
5497 MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5498 &AArch64::FPR128RegClass);
5499 MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5500 &AArch64::FPR128RegClass);
5509 }
else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
5510 !Subtarget.hasZeroCycleRegMoveFPR32()) {
5511 MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5512 &AArch64::FPR64RegClass);
5513 MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5514 &AArch64::FPR64RegClass);
5523 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
5524 &AArch64::FPR32RegClass);
5525 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
5526 &AArch64::FPR32RegClass);
5534 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5535 AArch64::GPR64RegClass.
contains(SrcReg)) {
5536 if (AArch64::XZR == SrcReg) {
5544 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5545 AArch64::FPR64RegClass.
contains(SrcReg)) {
5551 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5552 AArch64::GPR32RegClass.
contains(SrcReg)) {
5553 if (AArch64::WZR == SrcReg) {
5561 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5562 AArch64::FPR32RegClass.
contains(SrcReg)) {
5568 if (DestReg == AArch64::NZCV) {
5569 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5571 .
addImm(AArch64SysReg::NZCV)
5577 if (SrcReg == AArch64::NZCV) {
5578 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
5580 .
addImm(AArch64SysReg::NZCV)
5586 errs() << RI.getRegAsmName(DestReg) <<
" = COPY " << RI.getRegAsmName(SrcReg)
5597 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
5602 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
5604 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
5617 Register SrcReg,
bool isKill,
int FI,
5633 switch (
TRI->getSpillSize(*RC)) {
5635 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5636 Opc = AArch64::STRBui;
5639 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5640 Opc = AArch64::STRHui;
5641 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
5642 AArch64::PPRRegClass.hasSubClassEq(RC)) {
5643 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5644 "Unexpected register store without SVE store instructions");
5645 Opc = AArch64::STR_PXI;
5651 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5652 Opc = AArch64::STRWui;
5656 assert(SrcReg != AArch64::WSP);
5657 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5658 Opc = AArch64::STRSui;
5659 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5660 Opc = AArch64::STR_PPXI;
5665 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5666 Opc = AArch64::STRXui;
5670 assert(SrcReg != AArch64::SP);
5671 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5672 Opc = AArch64::STRDui;
5673 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5675 get(AArch64::STPWi), SrcReg, isKill,
5676 AArch64::sube32, AArch64::subo32, FI, MMO);
5681 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5682 Opc = AArch64::STRQui;
5683 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5684 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5685 Opc = AArch64::ST1Twov1d;
5687 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5689 get(AArch64::STPXi), SrcReg, isKill,
5690 AArch64::sube64, AArch64::subo64, FI, MMO);
5692 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5693 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5694 "Unexpected register store without SVE store instructions");
5695 Opc = AArch64::STR_ZXI;
5700 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5701 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5702 Opc = AArch64::ST1Threev1d;
5707 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5708 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5709 Opc = AArch64::ST1Fourv1d;
5711 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5712 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5713 Opc = AArch64::ST1Twov2d;
5715 }
else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5716 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5717 "Unexpected register store without SVE store instructions");
5718 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
5720 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
5721 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5722 "Unexpected register store without SVE store instructions");
5723 Opc = AArch64::STR_ZZXI;
5728 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5729 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5730 Opc = AArch64::ST1Threev2d;
5732 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5733 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5734 "Unexpected register store without SVE store instructions");
5735 Opc = AArch64::STR_ZZZXI;
5740 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5741 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5742 Opc = AArch64::ST1Fourv2d;
5744 }
else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5745 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5746 "Unexpected register store without SVE store instructions");
5747 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
5749 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
5750 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5751 "Unexpected register store without SVE store instructions");
5752 Opc = AArch64::STR_ZZZZXI;
5757 assert(
Opc &&
"Unknown register class");
5768 MI.addMemOperand(MMO);
5775 Register DestReg,
unsigned SubIdx0,
5776 unsigned SubIdx1,
int FI,
5780 bool IsUndef =
true;
5782 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
5784 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5811 switch (
TRI->getSpillSize(*RC)) {
5813 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5814 Opc = AArch64::LDRBui;
5817 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5818 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5819 Opc = AArch64::LDRHui;
5820 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5821 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5822 "Unexpected register load without SVE load instructions");
5825 Opc = AArch64::LDR_PXI;
5831 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5832 Opc = AArch64::LDRWui;
5836 assert(DestReg != AArch64::WSP);
5837 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5838 Opc = AArch64::LDRSui;
5839 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5840 Opc = AArch64::LDR_PPXI;
5845 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5846 Opc = AArch64::LDRXui;
5850 assert(DestReg != AArch64::SP);
5851 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5852 Opc = AArch64::LDRDui;
5853 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5855 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5856 AArch64::subo32, FI, MMO);
5861 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5862 Opc = AArch64::LDRQui;
5863 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5864 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5865 Opc = AArch64::LD1Twov1d;
5867 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5869 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5870 AArch64::subo64, FI, MMO);
5872 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5873 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5874 "Unexpected register load without SVE load instructions");
5875 Opc = AArch64::LDR_ZXI;
5880 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5881 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5882 Opc = AArch64::LD1Threev1d;
5887 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5888 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5889 Opc = AArch64::LD1Fourv1d;
5891 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5892 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5893 Opc = AArch64::LD1Twov2d;
5895 }
else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5896 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5897 "Unexpected register load without SVE load instructions");
5898 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
5900 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
5901 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5902 "Unexpected register load without SVE load instructions");
5903 Opc = AArch64::LDR_ZZXI;
5908 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5909 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5910 Opc = AArch64::LD1Threev2d;
5912 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5913 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5914 "Unexpected register load without SVE load instructions");
5915 Opc = AArch64::LDR_ZZZXI;
5920 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5921 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5922 Opc = AArch64::LD1Fourv2d;
5924 }
else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5925 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5926 "Unexpected register load without SVE load instructions");
5927 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
5929 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
5930 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5931 "Unexpected register load without SVE load instructions");
5932 Opc = AArch64::LDR_ZZZZXI;
5938 assert(
Opc &&
"Unknown register class");
5948 MI.addMemOperand(MMO);
5955 UseMI.getIterator()),
5957 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5958 I.readsRegister(AArch64::NZCV, TRI);
5962void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
5967 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5974 ByteSized =
Offset.getFixed();
5975 VGSized =
Offset.getScalable() / 2;
5981void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
5983 int64_t &NumDataVectors) {
5987 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5989 NumBytes =
Offset.getFixed();
5991 NumPredicateVectors =
Offset.getScalable() / 2;
5996 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5997 NumPredicateVectors > 62) {
5998 NumDataVectors = NumPredicateVectors / 8;
5999 NumPredicateVectors -= NumDataVectors * 8;
6025 Expr.
push_back((
char)dwarf::DW_OP_bregx);
6033 int64_t OffsetFromDefCFA) {
6047 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
6048 if (!RegScale.empty())
6058 int64_t NumBytes, NumVGScaledBytes;
6059 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
Offset, NumBytes,
6061 std::string CommentBuffer;
6064 if (
Reg == AArch64::SP)
6066 else if (
Reg == AArch64::FP)
6073 unsigned DwarfReg =
TRI.getDwarfRegNum(
Reg,
true);
6074 assert(DwarfReg <= 31 &&
"DwarfReg out of bounds (0..31)");
6076 Expr.
push_back(dwarf::DW_OP_breg0 + DwarfReg);
6079 if (NumVGScaledBytes) {
6089 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
6097 unsigned FrameReg,
unsigned Reg,
6099 bool LastAdjustmentWasScalable) {
6100 if (
Offset.getScalable())
6103 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6106 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
6113 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6114 int64_t NumBytes, NumVGScaledBytes;
6115 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6116 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
6118 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
6121 if (!NumVGScaledBytes)
6124 std::string CommentBuffer;
6129 assert(NumVGScaledBytes &&
"Expected scalable offset");
6133 if (IncomingVGOffsetFromDefCFA) {
6135 VGRegScale =
"* IncomingVG";
6138 VGRegScale =
"* VG";
6142 OffsetExpr.
push_back(dwarf::DW_OP_plus);
6151 CfaExpr.
push_back(dwarf::DW_CFA_expression);
6166 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
6169 bool *HasWinCFI,
bool EmitCFAOffset,
6172 unsigned MaxEncoding, ShiftSize;
6174 case AArch64::ADDXri:
6175 case AArch64::ADDSXri:
6176 case AArch64::SUBXri:
6177 case AArch64::SUBSXri:
6178 MaxEncoding = 0xfff;
6181 case AArch64::ADDVL_XXI:
6182 case AArch64::ADDPL_XXI:
6183 case AArch64::ADDSVL_XXI:
6184 case AArch64::ADDSPL_XXI:
6199 if (
Opc == AArch64::ADDVL_XXI ||
Opc == AArch64::ADDSVL_XXI)
6201 else if (
Opc == AArch64::ADDPL_XXI ||
Opc == AArch64::ADDSPL_XXI)
6215 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
6217 if (TmpReg == AArch64::XZR)
6218 TmpReg =
MBB.getParent()->getRegInfo().createVirtualRegister(
6219 &AArch64::GPR64RegClass);
6221 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
6222 unsigned LocalShiftSize = 0;
6223 if (ThisVal > MaxEncoding) {
6224 ThisVal = ThisVal >> ShiftSize;
6225 LocalShiftSize = ShiftSize;
6227 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
6228 "Encoding cannot handle value that big");
6230 Offset -= ThisVal << LocalShiftSize;
6235 .
addImm(Sign * (
int)ThisVal);
6245 if (Sign == -1 ||
Opc == AArch64::SUBXri ||
Opc == AArch64::SUBSXri)
6246 CFAOffset += Change;
6248 CFAOffset -= Change;
6249 if (EmitCFAOffset && DestReg == TmpReg) {
6262 int Imm = (int)(ThisVal << LocalShiftSize);
6263 if (VScale != 1 && DestReg == AArch64::SP) {
6269 }
else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
6270 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
6271 assert(VScale == 1 &&
"Expected non-scalable operation");
6280 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
6281 "emit a single SEH directive");
6282 }
else if (DestReg == AArch64::SP) {
6283 assert(VScale == 1 &&
"Expected non-scalable operation");
6286 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
6299 unsigned DestReg,
unsigned SrcReg,
6302 bool NeedsWinCFI,
bool *HasWinCFI,
6304 unsigned FrameReg) {
6311 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
6313 int64_t Bytes, NumPredicateVectors, NumDataVectors;
6314 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6315 Offset, Bytes, NumPredicateVectors, NumDataVectors);
6318 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
6319 if (NeedsFinalDefNZCV)
6323 if (Bytes || (!
Offset && SrcReg != DestReg)) {
6324 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
6325 "SP increment/decrement not 8-byte aligned");
6326 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
6329 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
6332 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6334 CFAOffset += (
Opc == AArch64::ADDXri ||
Opc == AArch64::ADDSXri)
6341 assert(!(NeedsWinCFI && NumPredicateVectors) &&
6342 "WinCFI can't allocate fractions of an SVE data vector");
6344 if (NumDataVectors) {
6346 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
TII,
6347 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6353 if (NumPredicateVectors) {
6354 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
6356 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
TII,
6357 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
6361 if (NeedsFinalDefNZCV)
6382 if (
MI.isFullCopy()) {
6385 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
6389 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
6394 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
6422 if (
MI.isCopy() &&
Ops.size() == 1 &&
6424 (
Ops[0] == 0 ||
Ops[0] == 1)) {
6425 bool IsSpill =
Ops[0] == 0;
6426 bool IsFill = !IsSpill;
6438 :
TRI.getMinimalPhysRegClass(Reg);
6444 "Mismatched register size in non subreg COPY");
6451 return &*--InsertPt;
6463 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
6466 "Unexpected subreg on physical register");
6468 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
6470 return &*--InsertPt;
6487 case AArch64::sub_32:
6488 if (AArch64::GPR64RegClass.hasSubClassEq(
getRegClass(DstReg)))
6489 FillRC = &AArch64::GPR32RegClass;
6492 FillRC = &AArch64::FPR32RegClass;
6495 FillRC = &AArch64::FPR64RegClass;
6501 TRI.getRegSizeInBits(*FillRC) &&
6502 "Mismatched regclass size on folded subreg COPY");
6521 bool *OutUseUnscaledOp,
6522 unsigned *OutUnscaledOp,
6523 int64_t *EmittableOffset) {
6525 if (EmittableOffset)
6526 *EmittableOffset = 0;
6527 if (OutUseUnscaledOp)
6528 *OutUseUnscaledOp =
false;
6534 switch (
MI.getOpcode()) {
6537 case AArch64::LD1Rv1d:
6538 case AArch64::LD1Rv2s:
6539 case AArch64::LD1Rv2d:
6540 case AArch64::LD1Rv4h:
6541 case AArch64::LD1Rv4s:
6542 case AArch64::LD1Rv8b:
6543 case AArch64::LD1Rv8h:
6544 case AArch64::LD1Rv16b:
6545 case AArch64::LD1Twov2d:
6546 case AArch64::LD1Threev2d:
6547 case AArch64::LD1Fourv2d:
6548 case AArch64::LD1Twov1d:
6549 case AArch64::LD1Threev1d:
6550 case AArch64::LD1Fourv1d:
6551 case AArch64::ST1Twov2d:
6552 case AArch64::ST1Threev2d:
6553 case AArch64::ST1Fourv2d:
6554 case AArch64::ST1Twov1d:
6555 case AArch64::ST1Threev1d:
6556 case AArch64::ST1Fourv1d:
6557 case AArch64::ST1i8:
6558 case AArch64::ST1i16:
6559 case AArch64::ST1i32:
6560 case AArch64::ST1i64:
6562 case AArch64::IRGstack:
6563 case AArch64::STGloop:
6564 case AArch64::STZGloop:
6569 TypeSize ScaleValue(0U,
false), Width(0U,
false);
6570 int64_t MinOff, MaxOff;
6576 bool IsMulVL = ScaleValue.isScalable();
6577 unsigned Scale = ScaleValue.getKnownMinValue();
6587 std::optional<unsigned> UnscaledOp =
6589 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
6590 if (useUnscaledOp &&
6595 Scale = ScaleValue.getKnownMinValue();
6596 assert(IsMulVL == ScaleValue.isScalable() &&
6597 "Unscaled opcode has different value for scalable");
6599 int64_t Remainder =
Offset % Scale;
6600 assert(!(Remainder && useUnscaledOp) &&
6601 "Cannot have remainder when using unscaled op");
6603 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
6604 int64_t NewOffset =
Offset / Scale;
6605 if (MinOff <= NewOffset && NewOffset <= MaxOff)
6608 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
6612 if (EmittableOffset)
6613 *EmittableOffset = NewOffset;
6614 if (OutUseUnscaledOp)
6615 *OutUseUnscaledOp = useUnscaledOp;
6616 if (OutUnscaledOp && UnscaledOp)
6617 *OutUnscaledOp = *UnscaledOp;
6630 unsigned Opcode =
MI.getOpcode();
6631 unsigned ImmIdx = FrameRegIdx + 1;
6633 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
6638 MI.eraseFromParent();
6644 unsigned UnscaledOp;
6647 &UnscaledOp, &NewOffset);
6651 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
6653 MI.setDesc(
TII->get(UnscaledOp));
6655 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
6673bool AArch64InstrInfo::useMachineCombiner()
const {
return true; }
6678 case AArch64::ADDSWrr:
6679 case AArch64::ADDSWri:
6680 case AArch64::ADDSXrr:
6681 case AArch64::ADDSXri:
6682 case AArch64::SUBSWrr:
6683 case AArch64::SUBSXrr:
6685 case AArch64::SUBSWri:
6686 case AArch64::SUBSXri:
6697 case AArch64::ADDWrr:
6698 case AArch64::ADDWri:
6699 case AArch64::SUBWrr:
6700 case AArch64::ADDSWrr:
6701 case AArch64::ADDSWri:
6702 case AArch64::SUBSWrr:
6704 case AArch64::SUBWri:
6705 case AArch64::SUBSWri:
6716 case AArch64::ADDXrr:
6717 case AArch64::ADDXri:
6718 case AArch64::SUBXrr:
6719 case AArch64::ADDSXrr:
6720 case AArch64::ADDSXri:
6721 case AArch64::SUBSXrr:
6723 case AArch64::SUBXri:
6724 case AArch64::SUBSXri:
6725 case AArch64::ADDv8i8:
6726 case AArch64::ADDv16i8:
6727 case AArch64::ADDv4i16:
6728 case AArch64::ADDv8i16:
6729 case AArch64::ADDv2i32:
6730 case AArch64::ADDv4i32:
6731 case AArch64::SUBv8i8:
6732 case AArch64::SUBv16i8:
6733 case AArch64::SUBv4i16:
6734 case AArch64::SUBv8i16:
6735 case AArch64::SUBv2i32:
6736 case AArch64::SUBv4i32:
6749 case AArch64::FADDHrr:
6750 case AArch64::FADDSrr:
6751 case AArch64::FADDDrr:
6752 case AArch64::FADDv4f16:
6753 case AArch64::FADDv8f16:
6754 case AArch64::FADDv2f32:
6755 case AArch64::FADDv2f64:
6756 case AArch64::FADDv4f32:
6757 case AArch64::FSUBHrr:
6758 case AArch64::FSUBSrr:
6759 case AArch64::FSUBDrr:
6760 case AArch64::FSUBv4f16:
6761 case AArch64::FSUBv8f16:
6762 case AArch64::FSUBv2f32:
6763 case AArch64::FSUBv2f64:
6764 case AArch64::FSUBv4f32:
6783 unsigned CombineOpc,
unsigned ZeroReg = 0,
6784 bool CheckZeroReg =
false) {
6791 if (!
MI ||
MI->getParent() != &
MBB ||
MI->getOpcode() != CombineOpc)
6794 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
6798 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
6799 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
6800 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
6802 if (
MI->getOperand(3).getReg() != ZeroReg)
6807 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
6816 unsigned MulOpc,
unsigned ZeroReg) {
6831bool AArch64InstrInfo::isAssociativeAndCommutative(
const MachineInstr &Inst,
6832 bool Invert)
const {
6838 case AArch64::FADDHrr:
6839 case AArch64::FADDSrr:
6840 case AArch64::FADDDrr:
6841 case AArch64::FMULHrr:
6842 case AArch64::FMULSrr:
6843 case AArch64::FMULDrr:
6844 case AArch64::FMULX16:
6845 case AArch64::FMULX32:
6846 case AArch64::FMULX64:
6848 case AArch64::FADDv4f16:
6849 case AArch64::FADDv8f16:
6850 case AArch64::FADDv2f32:
6851 case AArch64::FADDv4f32:
6852 case AArch64::FADDv2f64:
6853 case AArch64::FMULv4f16:
6854 case AArch64::FMULv8f16:
6855 case AArch64::FMULv2f32:
6856 case AArch64::FMULv4f32:
6857 case AArch64::FMULv2f64:
6858 case AArch64::FMULXv4f16:
6859 case AArch64::FMULXv8f16:
6860 case AArch64::FMULXv2f32:
6861 case AArch64::FMULXv4f32:
6862 case AArch64::FMULXv2f64:
6866 case AArch64::FADD_ZZZ_H:
6867 case AArch64::FADD_ZZZ_S:
6868 case AArch64::FADD_ZZZ_D:
6869 case AArch64::FMUL_ZZZ_H:
6870 case AArch64::FMUL_ZZZ_S:
6871 case AArch64::FMUL_ZZZ_D:
6882 case AArch64::ADDWrr:
6883 case AArch64::ADDXrr:
6884 case AArch64::ANDWrr:
6885 case AArch64::ANDXrr:
6886 case AArch64::ORRWrr:
6887 case AArch64::ORRXrr:
6888 case AArch64::EORWrr:
6889 case AArch64::EORXrr:
6890 case AArch64::EONWrr:
6891 case AArch64::EONXrr:
6895 case AArch64::ADDv8i8:
6896 case AArch64::ADDv16i8:
6897 case AArch64::ADDv4i16:
6898 case AArch64::ADDv8i16:
6899 case AArch64::ADDv2i32:
6900 case AArch64::ADDv4i32:
6901 case AArch64::ADDv1i64:
6902 case AArch64::ADDv2i64:
6903 case AArch64::MULv8i8:
6904 case AArch64::MULv16i8:
6905 case AArch64::MULv4i16:
6906 case AArch64::MULv8i16:
6907 case AArch64::MULv2i32:
6908 case AArch64::MULv4i32:
6909 case AArch64::ANDv8i8:
6910 case AArch64::ANDv16i8:
6911 case AArch64::ORRv8i8:
6912 case AArch64::ORRv16i8:
6913 case AArch64::EORv8i8:
6914 case AArch64::EORv16i8:
6916 case AArch64::ADD_ZZZ_B:
6917 case AArch64::ADD_ZZZ_H:
6918 case AArch64::ADD_ZZZ_S:
6919 case AArch64::ADD_ZZZ_D:
6920 case AArch64::MUL_ZZZ_B:
6921 case AArch64::MUL_ZZZ_H:
6922 case AArch64::MUL_ZZZ_S:
6923 case AArch64::MUL_ZZZ_D:
6924 case AArch64::AND_ZZZ:
6925 case AArch64::ORR_ZZZ:
6926 case AArch64::EOR_ZZZ:
6957 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6965 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6977 case AArch64::ADDWrr:
6979 "ADDWrr does not have register operands");
6980 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6981 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6983 case AArch64::ADDXrr:
6984 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6985 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6987 case AArch64::SUBWrr:
6988 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6989 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6991 case AArch64::SUBXrr:
6992 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6993 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6995 case AArch64::ADDWri:
6996 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6998 case AArch64::ADDXri:
6999 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7001 case AArch64::SUBWri:
7002 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7004 case AArch64::SUBXri:
7005 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7007 case AArch64::ADDv8i8:
7008 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7009 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7011 case AArch64::ADDv16i8:
7012 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7013 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7015 case AArch64::ADDv4i16:
7016 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7017 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7018 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7019 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7021 case AArch64::ADDv8i16:
7022 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7023 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7024 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7025 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7027 case AArch64::ADDv2i32:
7028 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7029 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7030 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7031 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7033 case AArch64::ADDv4i32:
7034 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7035 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7036 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7037 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7039 case AArch64::SUBv8i8:
7040 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7041 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7043 case AArch64::SUBv16i8:
7044 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7045 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7047 case AArch64::SUBv4i16:
7048 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7049 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7050 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7051 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7053 case AArch64::SUBv8i16:
7054 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7055 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7056 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7057 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7059 case AArch64::SUBv2i32:
7060 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7061 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7062 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7063 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7065 case AArch64::SUBv4i32:
7066 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7067 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7068 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7069 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7075bool AArch64InstrInfo::isAccumulationOpcode(
unsigned Opcode)
const {
7079 case AArch64::UABALB_ZZZ_D:
7080 case AArch64::UABALB_ZZZ_H:
7081 case AArch64::UABALB_ZZZ_S:
7082 case AArch64::UABALT_ZZZ_D:
7083 case AArch64::UABALT_ZZZ_H:
7084 case AArch64::UABALT_ZZZ_S:
7085 case AArch64::SABALB_ZZZ_D:
7086 case AArch64::SABALB_ZZZ_S:
7087 case AArch64::SABALB_ZZZ_H:
7088 case AArch64::SABALT_ZZZ_D:
7089 case AArch64::SABALT_ZZZ_S:
7090 case AArch64::SABALT_ZZZ_H:
7091 case AArch64::UABALv16i8_v8i16:
7092 case AArch64::UABALv2i32_v2i64:
7093 case AArch64::UABALv4i16_v4i32:
7094 case AArch64::UABALv4i32_v2i64:
7095 case AArch64::UABALv8i16_v4i32:
7096 case AArch64::UABALv8i8_v8i16:
7097 case AArch64::UABAv16i8:
7098 case AArch64::UABAv2i32:
7099 case AArch64::UABAv4i16:
7100 case AArch64::UABAv4i32:
7101 case AArch64::UABAv8i16:
7102 case AArch64::UABAv8i8:
7103 case AArch64::SABALv16i8_v8i16:
7104 case AArch64::SABALv2i32_v2i64:
7105 case AArch64::SABALv4i16_v4i32:
7106 case AArch64::SABALv4i32_v2i64:
7107 case AArch64::SABALv8i16_v4i32:
7108 case AArch64::SABALv8i8_v8i16:
7109 case AArch64::SABAv16i8:
7110 case AArch64::SABAv2i32:
7111 case AArch64::SABAv4i16:
7112 case AArch64::SABAv4i32:
7113 case AArch64::SABAv8i16:
7114 case AArch64::SABAv8i8:
7121unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7122 unsigned AccumulationOpcode)
const {
7123 switch (AccumulationOpcode) {
7126 case AArch64::UABALB_ZZZ_D:
7127 return AArch64::UABDLB_ZZZ_D;
7128 case AArch64::UABALB_ZZZ_H:
7129 return AArch64::UABDLB_ZZZ_H;
7130 case AArch64::UABALB_ZZZ_S:
7131 return AArch64::UABDLB_ZZZ_S;
7132 case AArch64::UABALT_ZZZ_D:
7133 return AArch64::UABDLT_ZZZ_D;
7134 case AArch64::UABALT_ZZZ_H:
7135 return AArch64::UABDLT_ZZZ_H;
7136 case AArch64::UABALT_ZZZ_S:
7137 return AArch64::UABDLT_ZZZ_S;
7138 case AArch64::UABALv16i8_v8i16:
7139 return AArch64::UABDLv16i8_v8i16;
7140 case AArch64::UABALv2i32_v2i64:
7141 return AArch64::UABDLv2i32_v2i64;
7142 case AArch64::UABALv4i16_v4i32:
7143 return AArch64::UABDLv4i16_v4i32;
7144 case AArch64::UABALv4i32_v2i64:
7145 return AArch64::UABDLv4i32_v2i64;
7146 case AArch64::UABALv8i16_v4i32:
7147 return AArch64::UABDLv8i16_v4i32;
7148 case AArch64::UABALv8i8_v8i16:
7149 return AArch64::UABDLv8i8_v8i16;
7150 case AArch64::UABAv16i8:
7151 return AArch64::UABDv16i8;
7152 case AArch64::UABAv2i32:
7153 return AArch64::UABDv2i32;
7154 case AArch64::UABAv4i16:
7155 return AArch64::UABDv4i16;
7156 case AArch64::UABAv4i32:
7157 return AArch64::UABDv4i32;
7158 case AArch64::UABAv8i16:
7159 return AArch64::UABDv8i16;
7160 case AArch64::UABAv8i8:
7161 return AArch64::UABDv8i8;
7162 case AArch64::SABALB_ZZZ_D:
7163 return AArch64::SABDLB_ZZZ_D;
7164 case AArch64::SABALB_ZZZ_S:
7165 return AArch64::SABDLB_ZZZ_S;
7166 case AArch64::SABALB_ZZZ_H:
7167 return AArch64::SABDLB_ZZZ_H;
7168 case AArch64::SABALT_ZZZ_D:
7169 return AArch64::SABDLT_ZZZ_D;
7170 case AArch64::SABALT_ZZZ_S:
7171 return AArch64::SABDLT_ZZZ_S;
7172 case AArch64::SABALT_ZZZ_H:
7173 return AArch64::SABDLT_ZZZ_H;
7174 case AArch64::SABALv16i8_v8i16:
7175 return AArch64::SABDLv16i8_v8i16;
7176 case AArch64::SABALv2i32_v2i64:
7177 return AArch64::SABDLv2i32_v2i64;
7178 case AArch64::SABALv4i16_v4i32:
7179 return AArch64::SABDLv4i16_v4i32;
7180 case AArch64::SABALv4i32_v2i64:
7181 return AArch64::SABDLv4i32_v2i64;
7182 case AArch64::SABALv8i16_v4i32:
7183 return AArch64::SABDLv8i16_v4i32;
7184 case AArch64::SABALv8i8_v8i16:
7185 return AArch64::SABDLv8i8_v8i16;
7186 case AArch64::SABAv16i8:
7187 return AArch64::SABDv16i8;
7188 case AArch64::SABAv2i32:
7189 return AArch64::SABAv2i32;
7190 case AArch64::SABAv4i16:
7191 return AArch64::SABDv4i16;
7192 case AArch64::SABAv4i32:
7193 return AArch64::SABDv4i32;
7194 case AArch64::SABAv8i16:
7195 return AArch64::SABDv8i16;
7196 case AArch64::SABAv8i8:
7197 return AArch64::SABDv8i8;
7213 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
7225 assert(
false &&
"Unsupported FP instruction in combiner\n");
7227 case AArch64::FADDHrr:
7229 "FADDHrr does not have register operands");
7231 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
7232 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
7234 case AArch64::FADDSrr:
7236 "FADDSrr does not have register operands");
7238 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
7239 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
7241 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
7242 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
7244 case AArch64::FADDDrr:
7245 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
7246 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
7248 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
7249 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
7251 case AArch64::FADDv4f16:
7252 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
7253 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
7255 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
7256 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
7258 case AArch64::FADDv8f16:
7259 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
7260 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
7262 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
7263 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
7265 case AArch64::FADDv2f32:
7266 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
7267 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
7269 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
7270 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
7272 case AArch64::FADDv2f64:
7273 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
7274 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
7276 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
7277 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
7279 case AArch64::FADDv4f32:
7280 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
7281 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
7283 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
7284 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
7286 case AArch64::FSUBHrr:
7287 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
7288 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
7289 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
7291 case AArch64::FSUBSrr:
7292 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
7294 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
7295 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
7297 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
7299 case AArch64::FSUBDrr:
7300 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
7302 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
7303 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
7305 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
7307 case AArch64::FSUBv4f16:
7308 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
7309 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
7311 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
7312 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
7314 case AArch64::FSUBv8f16:
7315 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
7316 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
7318 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
7319 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
7321 case AArch64::FSUBv2f32:
7322 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
7323 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
7325 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
7326 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
7328 case AArch64::FSUBv2f64:
7329 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
7330 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
7332 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
7333 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
7335 case AArch64::FSUBv4f32:
7336 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
7337 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
7339 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
7340 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
7351 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
7358 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
7359 MI->getOperand(1).getReg().isVirtual())
7360 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
7361 if (
MI &&
MI->getOpcode() == Opcode) {
7373 case AArch64::FMULv2f32:
7374 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
7375 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
7377 case AArch64::FMULv2f64:
7378 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
7379 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
7381 case AArch64::FMULv4f16:
7382 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
7383 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
7385 case AArch64::FMULv4f32:
7386 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
7387 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
7389 case AArch64::FMULv8f16:
7390 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
7391 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
7404 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
7407 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
7408 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
7422 case AArch64::FNEGDr:
7424 case AArch64::FNEGSr:
7556 case AArch64::SUBWrr:
7557 case AArch64::SUBSWrr:
7558 case AArch64::SUBXrr:
7559 case AArch64::SUBSXrr:
7604 unsigned LoadLaneOpCode,
unsigned NumLanes) {
7627 while (!RemainingLanes.
empty() && CurrInstr &&
7628 CurrInstr->getOpcode() == LoadLaneOpCode &&
7629 MRI.hasOneNonDBGUse(CurrInstr->getOperand(0).getReg()) &&
7630 CurrInstr->getNumOperands() == 4) {
7631 RemainingLanes.
erase(CurrInstr->getOperand(2).getImm());
7633 CurrInstr =
MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
7637 if (!RemainingLanes.
empty())
7641 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
7645 auto Lane0LoadReg = CurrInstr->getOperand(2).getReg();
7646 unsigned SingleLaneSizeInBits = 128 / NumLanes;
7647 if (
TRI->getRegSizeInBits(Lane0LoadReg,
MRI) != SingleLaneSizeInBits)
7651 if (!
MRI.hasOneNonDBGUse(Lane0LoadReg))
7654 LoadInstrs.
push_back(
MRI.getUniqueVRegDef(Lane0LoadReg));
7663 RemainingLoadInstrs.
insert(LoadInstrs.
begin(), LoadInstrs.
end());
7666 for (; MBBItr !=
MBB->begin() && RemainingSteps > 0 &&
7667 !RemainingLoadInstrs.
empty();
7668 --MBBItr, --RemainingSteps) {
7672 RemainingLoadInstrs.
erase(&CurrInstr);
7682 if (RemainingSteps == 0 && !RemainingLoadInstrs.
empty())
7708 case AArch64::LD1i32:
7710 case AArch64::LD1i16:
7712 case AArch64::LD1i8:
7728 unsigned Pattern,
unsigned NumLanes) {
7736 for (
unsigned i = 0; i < NumLanes - 1; ++i) {
7744 return A->getOperand(2).getImm() >
B->getOperand(2).getImm();
7749 MRI.getUniqueVRegDef(SubregToReg->getOperand(2).getReg()));
7750 auto LoadToLaneInstrsAscending =
llvm::reverse(LoadToLaneInstrs);
7756 auto CreateLD1Instruction = [&](
MachineInstr *OriginalInstr,
7757 Register SrcRegister,
unsigned Lane,
7759 bool OffsetRegisterKillState) {
7760 auto NewRegister =
MRI.createVirtualRegister(FPR128RegClass);
7767 InstrIdxForVirtReg.
insert(std::make_pair(NewRegister, InsInstrs.
size()));
7768 InsInstrs.
push_back(LoadIndexIntoRegister);
7774 auto CreateLDRInstruction = [&](
unsigned NumLanes,
Register DestReg,
7780 Opcode = AArch64::LDRSui;
7783 Opcode = AArch64::LDRHui;
7786 Opcode = AArch64::LDRBui;
7790 "Got unsupported number of lanes in machine-combiner gather pattern");
7799 auto LanesToLoadToReg0 =
7801 LoadToLaneInstrsAscending.begin() + NumLanes / 2);
7802 Register PrevReg = SubregToReg->getOperand(0).getReg();
7804 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
7805 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
7806 OffsetRegOperand.
getReg(),
7807 OffsetRegOperand.
isKill());
7814 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
7816 *std::next(LoadToLaneInstrsAscending.begin(), NumLanes / 2);
7817 Register DestRegForMiddleIndex =
MRI.createVirtualRegister(
7823 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
7824 OriginalSplitToLoadOffsetOperand.
getReg(),
7825 OriginalSplitToLoadOffsetOperand.
isKill());
7827 InstrIdxForVirtReg.
insert(
7828 std::make_pair(DestRegForMiddleIndex, InsInstrs.
size()));
7829 InsInstrs.
push_back(MiddleIndexLoadInstr);
7833 Register DestRegForSubregToReg =
MRI.createVirtualRegister(FPR128RegClass);
7834 unsigned SubregType;
7837 SubregType = AArch64::ssub;
7840 SubregType = AArch64::hsub;
7843 SubregType = AArch64::bsub;
7847 "Got invalid NumLanes for machine-combiner gather pattern");
7850 auto SubRegToRegInstr =
7852 DestRegForSubregToReg)
7856 InstrIdxForVirtReg.
insert(
7857 std::make_pair(DestRegForSubregToReg, InsInstrs.
size()));
7861 auto LanesToLoadToReg1 =
7863 LoadToLaneInstrsAscending.end());
7864 PrevReg = SubRegToRegInstr->getOperand(0).getReg();
7866 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
7867 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
7868 OffsetRegOperand.
getReg(),
7869 OffsetRegOperand.
isKill());
7872 if (Index == NumLanes / 2 - 2) {
7907bool AArch64InstrInfo::getMachineCombinerPatterns(
7909 bool DoRegPressureReduce)
const {
7930 DoRegPressureReduce);
7959 const Register *ReplacedAddend =
nullptr) {
7960 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7962 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
7965 Register SrcReg0 = MUL->getOperand(1).getReg();
7966 bool Src0IsKill = MUL->getOperand(1).isKill();
7967 Register SrcReg1 = MUL->getOperand(2).getReg();
7968 bool Src1IsKill = MUL->getOperand(2).isKill();
7972 if (ReplacedAddend) {
7974 SrcReg2 = *ReplacedAddend;
7982 MRI.constrainRegClass(ResultReg, RC);
7984 MRI.constrainRegClass(SrcReg0, RC);
7986 MRI.constrainRegClass(SrcReg1, RC);
7988 MRI.constrainRegClass(SrcReg2, RC);
8001 .
addImm(MUL->getOperand(3).getImm());
8008 assert(
false &&
"Invalid FMA instruction kind \n");
8022 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8023 Opc = AArch64::FNMADDSrrr;
8024 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8025 Opc = AArch64::FNMADDDrrr;
8037 MRI.constrainRegClass(ResultReg, RC);
8039 MRI.constrainRegClass(SrcReg0, RC);
8041 MRI.constrainRegClass(SrcReg1, RC);
8043 MRI.constrainRegClass(SrcReg2, RC);
8059 unsigned IdxDupOp,
unsigned MulOpc,
8061 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8062 "Invalid index of FMUL operand");
8070 if (Dup->
getOpcode() == TargetOpcode::COPY)
8074 MRI.clearKillFlags(DupSrcReg);
8075 MRI.constrainRegClass(DupSrcReg, RC);
8079 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8120 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8135 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8162 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8190 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
8192 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8196 Register SrcReg0 = MUL->getOperand(1).getReg();
8197 bool Src0IsKill = MUL->getOperand(1).isKill();
8198 Register SrcReg1 = MUL->getOperand(2).getReg();
8199 bool Src1IsKill = MUL->getOperand(2).isKill();
8202 MRI.constrainRegClass(ResultReg, RC);
8204 MRI.constrainRegClass(SrcReg0, RC);
8206 MRI.constrainRegClass(SrcReg1, RC);
8208 MRI.constrainRegClass(VR, RC);
8229 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
8230 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
8244 if (Opcode == AArch64::SUBSWrr)
8245 Opcode = AArch64::SUBWrr;
8246 else if (Opcode == AArch64::SUBSXrr)
8247 Opcode = AArch64::SUBXrr;
8249 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
8250 "Unexpected instruction opcode.");
8267 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8274unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
8275 unsigned int AccumulatorOpCode)
const {
8276 switch (AccumulatorOpCode) {
8277 case AArch64::UABALB_ZZZ_D:
8278 case AArch64::SABALB_ZZZ_D:
8279 case AArch64::UABALT_ZZZ_D:
8280 case AArch64::SABALT_ZZZ_D:
8281 return AArch64::ADD_ZZZ_D;
8282 case AArch64::UABALB_ZZZ_H:
8283 case AArch64::SABALB_ZZZ_H:
8284 case AArch64::UABALT_ZZZ_H:
8285 case AArch64::SABALT_ZZZ_H:
8286 return AArch64::ADD_ZZZ_H;
8287 case AArch64::UABALB_ZZZ_S:
8288 case AArch64::SABALB_ZZZ_S:
8289 case AArch64::UABALT_ZZZ_S:
8290 case AArch64::SABALT_ZZZ_S:
8291 return AArch64::ADD_ZZZ_S;
8292 case AArch64::UABALv16i8_v8i16:
8293 case AArch64::SABALv8i8_v8i16:
8294 case AArch64::SABAv8i16:
8295 case AArch64::UABAv8i16:
8296 return AArch64::ADDv8i16;
8297 case AArch64::SABALv2i32_v2i64:
8298 case AArch64::UABALv2i32_v2i64:
8299 case AArch64::SABALv4i32_v2i64:
8300 return AArch64::ADDv2i64;
8301 case AArch64::UABALv4i16_v4i32:
8302 case AArch64::SABALv4i16_v4i32:
8303 case AArch64::SABALv8i16_v4i32:
8304 case AArch64::SABAv4i32:
8305 case AArch64::UABAv4i32:
8306 return AArch64::ADDv4i32;
8307 case AArch64::UABALv4i32_v2i64:
8308 return AArch64::ADDv2i64;
8309 case AArch64::UABALv8i16_v4i32:
8310 return AArch64::ADDv4i32;
8311 case AArch64::UABALv8i8_v8i16:
8312 case AArch64::SABALv16i8_v8i16:
8313 return AArch64::ADDv8i16;
8314 case AArch64::UABAv16i8:
8315 case AArch64::SABAv16i8:
8316 return AArch64::ADDv16i8;
8317 case AArch64::UABAv4i16:
8318 case AArch64::SABAv4i16:
8319 return AArch64::ADDv4i16;
8320 case AArch64::UABAv2i32:
8321 case AArch64::SABAv2i32:
8322 return AArch64::ADDv2i32;
8323 case AArch64::UABAv8i8:
8324 case AArch64::SABAv8i8:
8325 return AArch64::ADDv8i8;
8334void AArch64InstrInfo::genAlternativeCodeSequence(
8344 MachineInstr *
MUL =
nullptr;
8345 const TargetRegisterClass *RC;
8351 DelInstrs, InstrIdxForVirtReg);
8357 InstrIdxForVirtReg);
8363 InstrIdxForVirtReg);
8372 Opc = AArch64::MADDWrrr;
8373 RC = &AArch64::GPR32RegClass;
8375 Opc = AArch64::MADDXrrr;
8376 RC = &AArch64::GPR64RegClass;
8387 Opc = AArch64::MADDWrrr;
8388 RC = &AArch64::GPR32RegClass;
8390 Opc = AArch64::MADDXrrr;
8391 RC = &AArch64::GPR64RegClass;
8404 const TargetRegisterClass *RC;
8405 unsigned BitSize, MovImm;
8408 MovImm = AArch64::MOVi32imm;
8409 RC = &AArch64::GPR32spRegClass;
8411 Opc = AArch64::MADDWrrr;
8412 RC = &AArch64::GPR32RegClass;
8414 MovImm = AArch64::MOVi64imm;
8415 RC = &AArch64::GPR64spRegClass;
8417 Opc = AArch64::MADDXrrr;
8418 RC = &AArch64::GPR64RegClass;
8429 uint64_t UImm =
SignExtend64(IsSub ? -Imm : Imm, BitSize);
8433 if (Insn.
size() != 1)
8435 MachineInstrBuilder MIB1 =
8436 BuildMI(MF, MIMetadata(Root),
TII->get(MovImm), NewVR)
8437 .
addImm(IsSub ? -Imm : Imm);
8439 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8450 const TargetRegisterClass *SubRC;
8451 unsigned SubOpc, ZeroReg;
8453 SubOpc = AArch64::SUBWrr;
8454 SubRC = &AArch64::GPR32spRegClass;
8455 ZeroReg = AArch64::WZR;
8456 Opc = AArch64::MADDWrrr;
8457 RC = &AArch64::GPR32RegClass;
8459 SubOpc = AArch64::SUBXrr;
8460 SubRC = &AArch64::GPR64spRegClass;
8461 ZeroReg = AArch64::XZR;
8462 Opc = AArch64::MADDXrrr;
8463 RC = &AArch64::GPR64RegClass;
8465 Register NewVR =
MRI.createVirtualRegister(SubRC);
8467 MachineInstrBuilder MIB1 =
8468 BuildMI(MF, MIMetadata(Root),
TII->get(SubOpc), NewVR)
8472 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8483 Opc = AArch64::MSUBWrrr;
8484 RC = &AArch64::GPR32RegClass;
8486 Opc = AArch64::MSUBXrrr;
8487 RC = &AArch64::GPR64RegClass;
8492 Opc = AArch64::MLAv8i8;
8493 RC = &AArch64::FPR64RegClass;
8497 Opc = AArch64::MLAv8i8;
8498 RC = &AArch64::FPR64RegClass;
8502 Opc = AArch64::MLAv16i8;
8503 RC = &AArch64::FPR128RegClass;
8507 Opc = AArch64::MLAv16i8;
8508 RC = &AArch64::FPR128RegClass;
8512 Opc = AArch64::MLAv4i16;
8513 RC = &AArch64::FPR64RegClass;
8517 Opc = AArch64::MLAv4i16;
8518 RC = &AArch64::FPR64RegClass;
8522 Opc = AArch64::MLAv8i16;
8523 RC = &AArch64::FPR128RegClass;
8527 Opc = AArch64::MLAv8i16;
8528 RC = &AArch64::FPR128RegClass;
8532 Opc = AArch64::MLAv2i32;
8533 RC = &AArch64::FPR64RegClass;
8537 Opc = AArch64::MLAv2i32;
8538 RC = &AArch64::FPR64RegClass;
8542 Opc = AArch64::MLAv4i32;
8543 RC = &AArch64::FPR128RegClass;
8547 Opc = AArch64::MLAv4i32;
8548 RC = &AArch64::FPR128RegClass;
8553 Opc = AArch64::MLAv8i8;
8554 RC = &AArch64::FPR64RegClass;
8556 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i8,
8560 Opc = AArch64::MLSv8i8;
8561 RC = &AArch64::FPR64RegClass;
8565 Opc = AArch64::MLAv16i8;
8566 RC = &AArch64::FPR128RegClass;
8568 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv16i8,
8572 Opc = AArch64::MLSv16i8;
8573 RC = &AArch64::FPR128RegClass;
8577 Opc = AArch64::MLAv4i16;
8578 RC = &AArch64::FPR64RegClass;
8580 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i16,
8584 Opc = AArch64::MLSv4i16;
8585 RC = &AArch64::FPR64RegClass;
8589 Opc = AArch64::MLAv8i16;
8590 RC = &AArch64::FPR128RegClass;
8592 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i16,
8596 Opc = AArch64::MLSv8i16;
8597 RC = &AArch64::FPR128RegClass;
8601 Opc = AArch64::MLAv2i32;
8602 RC = &AArch64::FPR64RegClass;
8604 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv2i32,
8608 Opc = AArch64::MLSv2i32;
8609 RC = &AArch64::FPR64RegClass;
8613 Opc = AArch64::MLAv4i32;
8614 RC = &AArch64::FPR128RegClass;
8616 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i32,
8620 Opc = AArch64::MLSv4i32;
8621 RC = &AArch64::FPR128RegClass;
8626 Opc = AArch64::MLAv4i16_indexed;
8627 RC = &AArch64::FPR64RegClass;
8631 Opc = AArch64::MLAv4i16_indexed;
8632 RC = &AArch64::FPR64RegClass;
8636 Opc = AArch64::MLAv8i16_indexed;
8637 RC = &AArch64::FPR128RegClass;
8641 Opc = AArch64::MLAv8i16_indexed;
8642 RC = &AArch64::FPR128RegClass;
8646 Opc = AArch64::MLAv2i32_indexed;
8647 RC = &AArch64::FPR64RegClass;
8651 Opc = AArch64::MLAv2i32_indexed;
8652 RC = &AArch64::FPR64RegClass;
8656 Opc = AArch64::MLAv4i32_indexed;
8657 RC = &AArch64::FPR128RegClass;
8661 Opc = AArch64::MLAv4i32_indexed;
8662 RC = &AArch64::FPR128RegClass;
8667 Opc = AArch64::MLAv4i16_indexed;
8668 RC = &AArch64::FPR64RegClass;
8670 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i16,
8674 Opc = AArch64::MLSv4i16_indexed;
8675 RC = &AArch64::FPR64RegClass;
8679 Opc = AArch64::MLAv8i16_indexed;
8680 RC = &AArch64::FPR128RegClass;
8682 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv8i16,
8686 Opc = AArch64::MLSv8i16_indexed;
8687 RC = &AArch64::FPR128RegClass;
8691 Opc = AArch64::MLAv2i32_indexed;
8692 RC = &AArch64::FPR64RegClass;
8694 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv2i32,
8698 Opc = AArch64::MLSv2i32_indexed;
8699 RC = &AArch64::FPR64RegClass;
8703 Opc = AArch64::MLAv4i32_indexed;
8704 RC = &AArch64::FPR128RegClass;
8706 InstrIdxForVirtReg, 1,
Opc, AArch64::NEGv4i32,
8710 Opc = AArch64::MLSv4i32_indexed;
8711 RC = &AArch64::FPR128RegClass;
8717 Opc = AArch64::FMADDHrrr;
8718 RC = &AArch64::FPR16RegClass;
8722 Opc = AArch64::FMADDSrrr;
8723 RC = &AArch64::FPR32RegClass;
8727 Opc = AArch64::FMADDDrrr;
8728 RC = &AArch64::FPR64RegClass;
8733 Opc = AArch64::FMADDHrrr;
8734 RC = &AArch64::FPR16RegClass;
8738 Opc = AArch64::FMADDSrrr;
8739 RC = &AArch64::FPR32RegClass;
8743 Opc = AArch64::FMADDDrrr;
8744 RC = &AArch64::FPR64RegClass;
8749 Opc = AArch64::FMLAv1i32_indexed;
8750 RC = &AArch64::FPR32RegClass;
8755 Opc = AArch64::FMLAv1i32_indexed;
8756 RC = &AArch64::FPR32RegClass;
8762 Opc = AArch64::FMLAv1i64_indexed;
8763 RC = &AArch64::FPR64RegClass;
8768 Opc = AArch64::FMLAv1i64_indexed;
8769 RC = &AArch64::FPR64RegClass;
8775 RC = &AArch64::FPR64RegClass;
8776 Opc = AArch64::FMLAv4i16_indexed;
8781 RC = &AArch64::FPR64RegClass;
8782 Opc = AArch64::FMLAv4f16;
8787 RC = &AArch64::FPR64RegClass;
8788 Opc = AArch64::FMLAv4i16_indexed;
8793 RC = &AArch64::FPR64RegClass;
8794 Opc = AArch64::FMLAv4f16;
8801 RC = &AArch64::FPR64RegClass;
8803 Opc = AArch64::FMLAv2i32_indexed;
8807 Opc = AArch64::FMLAv2f32;
8814 RC = &AArch64::FPR64RegClass;
8816 Opc = AArch64::FMLAv2i32_indexed;
8820 Opc = AArch64::FMLAv2f32;
8827 RC = &AArch64::FPR128RegClass;
8828 Opc = AArch64::FMLAv8i16_indexed;
8833 RC = &AArch64::FPR128RegClass;
8834 Opc = AArch64::FMLAv8f16;
8839 RC = &AArch64::FPR128RegClass;
8840 Opc = AArch64::FMLAv8i16_indexed;
8845 RC = &AArch64::FPR128RegClass;
8846 Opc = AArch64::FMLAv8f16;
8853 RC = &AArch64::FPR128RegClass;
8855 Opc = AArch64::FMLAv2i64_indexed;
8859 Opc = AArch64::FMLAv2f64;
8866 RC = &AArch64::FPR128RegClass;
8868 Opc = AArch64::FMLAv2i64_indexed;
8872 Opc = AArch64::FMLAv2f64;
8880 RC = &AArch64::FPR128RegClass;
8882 Opc = AArch64::FMLAv4i32_indexed;
8886 Opc = AArch64::FMLAv4f32;
8894 RC = &AArch64::FPR128RegClass;
8896 Opc = AArch64::FMLAv4i32_indexed;
8900 Opc = AArch64::FMLAv4f32;
8907 Opc = AArch64::FNMSUBHrrr;
8908 RC = &AArch64::FPR16RegClass;
8912 Opc = AArch64::FNMSUBSrrr;
8913 RC = &AArch64::FPR32RegClass;
8917 Opc = AArch64::FNMSUBDrrr;
8918 RC = &AArch64::FPR64RegClass;
8923 Opc = AArch64::FNMADDHrrr;
8924 RC = &AArch64::FPR16RegClass;
8928 Opc = AArch64::FNMADDSrrr;
8929 RC = &AArch64::FPR32RegClass;
8933 Opc = AArch64::FNMADDDrrr;
8934 RC = &AArch64::FPR64RegClass;
8939 Opc = AArch64::FMSUBHrrr;
8940 RC = &AArch64::FPR16RegClass;
8944 Opc = AArch64::FMSUBSrrr;
8945 RC = &AArch64::FPR32RegClass;
8949 Opc = AArch64::FMSUBDrrr;
8950 RC = &AArch64::FPR64RegClass;
8955 Opc = AArch64::FMLSv1i32_indexed;
8956 RC = &AArch64::FPR32RegClass;
8962 Opc = AArch64::FMLSv1i64_indexed;
8963 RC = &AArch64::FPR64RegClass;
8970 RC = &AArch64::FPR64RegClass;
8972 MachineInstrBuilder MIB1 =
8973 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv4f16), NewVR)
8976 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8978 Opc = AArch64::FMLAv4f16;
8982 Opc = AArch64::FMLAv4i16_indexed;
8989 RC = &AArch64::FPR64RegClass;
8990 Opc = AArch64::FMLSv4f16;
8995 RC = &AArch64::FPR64RegClass;
8996 Opc = AArch64::FMLSv4i16_indexed;
9003 RC = &AArch64::FPR64RegClass;
9005 Opc = AArch64::FMLSv2i32_indexed;
9009 Opc = AArch64::FMLSv2f32;
9017 RC = &AArch64::FPR128RegClass;
9019 MachineInstrBuilder MIB1 =
9020 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv8f16), NewVR)
9023 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9025 Opc = AArch64::FMLAv8f16;
9029 Opc = AArch64::FMLAv8i16_indexed;
9036 RC = &AArch64::FPR128RegClass;
9037 Opc = AArch64::FMLSv8f16;
9042 RC = &AArch64::FPR128RegClass;
9043 Opc = AArch64::FMLSv8i16_indexed;
9050 RC = &AArch64::FPR128RegClass;
9052 Opc = AArch64::FMLSv2i64_indexed;
9056 Opc = AArch64::FMLSv2f64;
9064 RC = &AArch64::FPR128RegClass;
9066 Opc = AArch64::FMLSv4i32_indexed;
9070 Opc = AArch64::FMLSv4f32;
9077 RC = &AArch64::FPR64RegClass;
9079 MachineInstrBuilder MIB1 =
9080 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv2f32), NewVR)
9083 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9085 Opc = AArch64::FMLAv2i32_indexed;
9089 Opc = AArch64::FMLAv2f32;
9097 RC = &AArch64::FPR128RegClass;
9099 MachineInstrBuilder MIB1 =
9100 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv4f32), NewVR)
9103 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9105 Opc = AArch64::FMLAv4i32_indexed;
9109 Opc = AArch64::FMLAv4f32;
9117 RC = &AArch64::FPR128RegClass;
9119 MachineInstrBuilder MIB1 =
9120 BuildMI(MF, MIMetadata(Root),
TII->get(AArch64::FNEGv2f64), NewVR)
9123 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
9125 Opc = AArch64::FMLAv2i64_indexed;
9129 Opc = AArch64::FMLAv2f64;
9141 &AArch64::FPR128RegClass,
MRI);
9150 &AArch64::FPR128RegClass,
MRI);
9159 &AArch64::FPR128_loRegClass,
MRI);
9168 &AArch64::FPR128RegClass,
MRI);
9177 &AArch64::FPR128_loRegClass,
MRI);
9211 for (
auto *
MI : InsInstrs)
9212 MI->setFlags(Flags);
9253 bool IsNegativeBranch =
false;
9254 bool IsTestAndBranch =
false;
9255 unsigned TargetBBInMI = 0;
9256 switch (
MI.getOpcode()) {
9260 case AArch64::CBWPri:
9261 case AArch64::CBXPri:
9262 case AArch64::CBWPrr:
9263 case AArch64::CBXPrr:
9269 case AArch64::CBNZW:
9270 case AArch64::CBNZX:
9272 IsNegativeBranch =
true;
9277 IsTestAndBranch =
true;
9279 case AArch64::TBNZW:
9280 case AArch64::TBNZX:
9282 IsNegativeBranch =
true;
9283 IsTestAndBranch =
true;
9289 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
9293 assert(
MI.getParent() &&
"Incomplete machine instruction\n");
9304 while (
DefMI->isCopy()) {
9306 if (!
MRI->hasOneNonDBGUse(CopyVReg))
9308 if (!
MRI->hasOneDef(CopyVReg))
9313 switch (
DefMI->getOpcode()) {
9317 case AArch64::ANDWri:
9318 case AArch64::ANDXri: {
9319 if (IsTestAndBranch)
9323 if (!
MRI->hasOneNonDBGUse(VReg))
9326 bool Is32Bit = (
DefMI->getOpcode() == AArch64::ANDWri);
9328 DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64);
9337 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
9343 unsigned Opc = (Imm < 32)
9344 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
9345 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
9358 if (!Is32Bit && Imm < 32)
9360 MI.eraseFromParent();
9364 case AArch64::CSINCWr:
9365 case AArch64::CSINCXr: {
9366 if (!(
DefMI->getOperand(1).getReg() == AArch64::WZR &&
9367 DefMI->getOperand(2).getReg() == AArch64::WZR) &&
9368 !(
DefMI->getOperand(1).getReg() == AArch64::XZR &&
9369 DefMI->getOperand(2).getReg() == AArch64::XZR))
9372 if (
DefMI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
9385 if (IsNegativeBranch)
9388 MI.eraseFromParent();
9394std::pair<unsigned, unsigned>
9395AArch64InstrInfo::decomposeMachineOperandsTargetFlags(
unsigned TF)
const {
9397 return std::make_pair(TF & Mask, TF & ~Mask);
9401AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags()
const {
9404 static const std::pair<unsigned, const char *> TargetFlags[] = {
9405 {MO_PAGE,
"aarch64-page"}, {
MO_PAGEOFF,
"aarch64-pageoff"},
9406 {
MO_G3,
"aarch64-g3"}, {
MO_G2,
"aarch64-g2"},
9407 {
MO_G1,
"aarch64-g1"}, {
MO_G0,
"aarch64-g0"},
9413AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags()
const {
9414 using namespace AArch64II;
9416 static const std::pair<unsigned, const char *> TargetFlags[] = {
9419 {
MO_NC,
"aarch64-nc"},
9420 {
MO_S,
"aarch64-s"},
9431AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags()
const {
9432 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
9534 MachineFunction *MF =
C.getMF();
9536 const AArch64RegisterInfo *ARI =
9537 static_cast<const AArch64RegisterInfo *
>(&
TRI);
9540 for (
unsigned Reg : AArch64::GPR64RegClass) {
9542 Reg != AArch64::LR &&
9543 Reg != AArch64::X16 &&
9544 Reg != AArch64::X17 &&
9545 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
9546 C.isAvailableInsideSeq(
Reg,
TRI))
9577 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
9580std::optional<std::unique_ptr<outliner::OutlinedFunction>>
9581AArch64InstrInfo::getOutliningCandidateInfo(
9583 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
9584 unsigned MinRepeats)
const {
9585 unsigned SequenceSize = 0;
9586 for (
auto &
MI : RepeatedSequenceLocs[0])
9589 unsigned NumBytesToCreateFrame = 0;
9599 if (std::adjacent_find(
9600 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
9601 [](
const outliner::Candidate &a,
const outliner::Candidate &b) {
9604 if (outliningCandidatesSigningScopeConsensus(a, b) &&
9605 outliningCandidatesSigningKeyConsensus(a, b) &&
9606 outliningCandidatesV8_3OpsConsensus(a, b)) {
9610 }) != RepeatedSequenceLocs.end()) {
9611 return std::nullopt;
9628 unsigned NumBytesToCheckLRInTCEpilogue = 0;
9629 if (RepeatedSequenceLocs[0]
9631 ->getInfo<AArch64FunctionInfo>()
9632 ->shouldSignReturnAddress(
true)) {
9634 NumBytesToCreateFrame += 8;
9637 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
9638 *RepeatedSequenceLocs[0].getMF());
9639 NumBytesToCheckLRInTCEpilogue =
9643 if (isTailCallReturnInst(RepeatedSequenceLocs[0].
back()))
9644 SequenceSize += NumBytesToCheckLRInTCEpilogue;
9652 for (
auto &
MI :
C) {
9653 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
9654 switch (
MI.getOpcode()) {
9655 case AArch64::ADDXri:
9656 case AArch64::ADDWri:
9657 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
9659 "Expected operand to be immediate");
9661 "Expected operand to be a register");
9665 if (
MI.getOperand(1).getReg() == AArch64::SP)
9666 SPValue +=
MI.getOperand(2).getImm();
9670 case AArch64::SUBXri:
9671 case AArch64::SUBWri:
9672 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
9674 "Expected operand to be immediate");
9676 "Expected operand to be a register");
9680 if (
MI.getOperand(1).getReg() == AArch64::SP)
9681 SPValue -=
MI.getOperand(2).getImm();
9698 if (RepeatedSequenceLocs.size() < MinRepeats)
9699 return std::nullopt;
9703 unsigned FlagsSetInAll = 0xF;
9707 FlagsSetInAll &=
C.Flags;
9709 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
9712 auto SetCandidateCallInfo =
9713 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
9715 C.setCallInfo(CallID, NumBytesForCall);
9719 NumBytesToCreateFrame += 4;
9727 unsigned CFICount = 0;
9728 for (
auto &
I : RepeatedSequenceLocs[0]) {
9729 if (
I.isCFIInstruction())
9739 std::vector<MCCFIInstruction> CFIInstructions =
9740 C.getMF()->getFrameInstructions();
9742 if (CFICount > 0 && CFICount != CFIInstructions.size())
9743 return std::nullopt;
9751 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
9752 !
MI.readsRegister(AArch64::SP, &
TRI))
9758 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
9763 if (
MI.mayLoadOrStore()) {
9766 bool OffsetIsScalable;
9770 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
9771 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
9775 if (OffsetIsScalable)
9783 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
9784 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
9787 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
9788 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
9803 bool AllStackInstrsSafe =
9808 if (RepeatedSequenceLocs[0].
back().isTerminator()) {
9810 NumBytesToCreateFrame = 0;
9811 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
9815 else if (LastInstrOpcode == AArch64::BL ||
9816 ((LastInstrOpcode == AArch64::BLR ||
9817 LastInstrOpcode == AArch64::BLRNoIP) &&
9821 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
9829 unsigned NumBytesNoStackCalls = 0;
9830 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
9836 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
9845 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
9848 if (LRAvailable && !IsNoReturn) {
9849 NumBytesNoStackCalls += 4;
9851 CandidatesWithoutStackFixups.push_back(
C);
9856 else if (findRegisterToSaveLRTo(
C)) {
9857 NumBytesNoStackCalls += 12;
9859 CandidatesWithoutStackFixups.push_back(
C);
9864 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
9865 NumBytesNoStackCalls += 12;
9867 CandidatesWithoutStackFixups.push_back(
C);
9873 NumBytesNoStackCalls += SequenceSize;
9880 if (!AllStackInstrsSafe ||
9881 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
9882 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
9884 if (RepeatedSequenceLocs.size() < MinRepeats)
9885 return std::nullopt;
9938 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
9939 !findRegisterToSaveLRTo(
C));
9945 if (RepeatedSequenceLocs.size() < MinRepeats)
9946 return std::nullopt;
9955 bool ModStackToSaveLR =
false;
9958 ModStackToSaveLR =
true;
9967 ModStackToSaveLR =
true;
9969 if (ModStackToSaveLR) {
9971 if (!AllStackInstrsSafe)
9972 return std::nullopt;
9975 NumBytesToCreateFrame += 8;
9982 return std::nullopt;
9984 return std::make_unique<outliner::OutlinedFunction>(
9985 RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID);
9988void AArch64InstrInfo::mergeOutliningCandidateAttributes(
9989 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
9993 const auto &CFn = Candidates.front().getMF()->getFunction();
9995 if (CFn.hasFnAttribute(
"ptrauth-returns"))
9996 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-returns"));
9997 if (CFn.hasFnAttribute(
"ptrauth-auth-traps"))
9998 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-auth-traps"));
10001 if (CFn.hasFnAttribute(
"sign-return-address"))
10002 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
10003 if (CFn.hasFnAttribute(
"sign-return-address-key"))
10004 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
10006 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
10009bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10014 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
10021 if (
F.hasSection())
10027 AArch64FunctionInfo *AFI = MF.
getInfo<AArch64FunctionInfo>();
10028 if (!AFI || AFI->
hasRedZone().value_or(
true))
10048 unsigned &Flags)
const {
10050 "Must track liveness!");
10052 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10067 auto AreAllUnsafeRegsDead = [&LRU]() {
10068 return LRU.available(AArch64::W16) && LRU.available(AArch64::W17) &&
10069 LRU.available(AArch64::NZCV);
10084 bool LRAvailableEverywhere =
true;
10086 LRU.addLiveOuts(
MBB);
10088 auto UpdateWholeMBBFlags = [&
Flags](
const MachineInstr &
MI) {
10089 if (
MI.isCall() && !
MI.isTerminator())
10095 auto CreateNewRangeStartingAt =
10096 [&RangeBegin, &RangeEnd,
10098 RangeBegin = NewBegin;
10099 RangeEnd = std::next(RangeBegin);
10102 auto SaveRangeIfNonEmpty = [&RangeLen, &
Ranges, &RangeBegin, &RangeEnd]() {
10108 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
10110 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
10112 Ranges.emplace_back(RangeBegin, RangeEnd);
10120 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
10121 LRU.stepBackward(*FirstPossibleEndPt);
10124 UpdateWholeMBBFlags(*FirstPossibleEndPt);
10125 if (AreAllUnsafeRegsDead())
10132 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
10137 LRU.stepBackward(
MI);
10138 UpdateWholeMBBFlags(
MI);
10139 if (!AreAllUnsafeRegsDead()) {
10140 SaveRangeIfNonEmpty();
10141 CreateNewRangeStartingAt(
MI.getIterator());
10144 LRAvailableEverywhere &= LRU.available(AArch64::LR);
10145 RangeBegin =
MI.getIterator();
10150 if (AreAllUnsafeRegsDead())
10151 SaveRangeIfNonEmpty();
10159 if (!LRAvailableEverywhere)
10167 unsigned Flags)
const {
10168 MachineInstr &
MI = *MIT;
10172 switch (
MI.getOpcode()) {
10173 case AArch64::PACM:
10174 case AArch64::PACIASP:
10175 case AArch64::PACIBSP:
10176 case AArch64::PACIASPPC:
10177 case AArch64::PACIBSPPC:
10178 case AArch64::AUTIASP:
10179 case AArch64::AUTIBSP:
10180 case AArch64::AUTIASPPCi:
10181 case AArch64::AUTIASPPCr:
10182 case AArch64::AUTIBSPPCi:
10183 case AArch64::AUTIBSPPCr:
10184 case AArch64::RETAA:
10185 case AArch64::RETAB:
10186 case AArch64::RETAASPPCi:
10187 case AArch64::RETAASPPCr:
10188 case AArch64::RETABSPPCi:
10189 case AArch64::RETABSPPCr:
10190 case AArch64::EMITBKEY:
10191 case AArch64::PAUTH_PROLOGUE:
10192 case AArch64::PAUTH_EPILOGUE:
10202 if (
MI.isCFIInstruction())
10206 if (
MI.isTerminator())
10212 for (
const MachineOperand &MOP :
MI.operands()) {
10215 assert(!MOP.isCFIIndex());
10218 if (MOP.isReg() && !MOP.isImplicit() &&
10219 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
10226 if (
MI.getOpcode() == AArch64::ADRP)
10246 for (
const MachineOperand &MOP :
MI.operands()) {
10247 if (MOP.isGlobal()) {
10255 if (Callee &&
Callee->getName() ==
"\01_mcount")
10263 if (
MI.getOpcode() == AArch64::BLR ||
10264 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
10268 return UnknownCallOutlineType;
10276 return UnknownCallOutlineType;
10284 return UnknownCallOutlineType;
10305 for (MachineInstr &
MI :
MBB) {
10306 const MachineOperand *
Base;
10307 TypeSize Width(0,
false);
10309 bool OffsetIsScalable;
10312 if (!
MI.mayLoadOrStore() ||
10315 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
10319 TypeSize Scale(0U,
false);
10320 int64_t Dummy1, Dummy2;
10323 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
10325 assert(Scale != 0 &&
"Unexpected opcode!");
10326 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
10331 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
10332 StackOffsetOperand.
setImm(NewImm);
10338 bool ShouldSignReturnAddr) {
10339 if (!ShouldSignReturnAddr)
10345 TII->get(AArch64::PAUTH_EPILOGUE))
10349void AArch64InstrInfo::buildOutlinedFrame(
10353 AArch64FunctionInfo *FI = MF.
getInfo<AArch64FunctionInfo>();
10361 unsigned TailOpcode;
10363 TailOpcode = AArch64::TCRETURNdi;
10367 TailOpcode = AArch64::TCRETURNriALL;
10378 bool IsLeafFunction =
true;
10381 auto IsNonTailCall = [](
const MachineInstr &
MI) {
10382 return MI.isCall() && !
MI.isReturn();
10392 "Can only fix up stack references once");
10393 fixupPostOutline(
MBB);
10395 IsLeafFunction =
false;
10406 Et = std::prev(
MBB.
end());
10416 if (MF.
getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
10420 CFIBuilder.buildDefCFAOffset(16);
10424 CFIBuilder.buildOffset(AArch64::LR, -16);
10465 fixupPostOutline(
MBB);
10476 .addGlobalAddress(
M.getNamedValue(MF.
getName()))
10486 .addGlobalAddress(
M.getNamedValue(MF.
getName())));
10495 MachineInstr *Save;
10496 MachineInstr *Restore;
10502 assert(
Reg &&
"No callee-saved register available?");
10536 .addGlobalAddress(
M.getNamedValue(MF.
getName())));
10544bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
10552 bool AllowSideEffects)
const {
10554 const AArch64Subtarget &STI = MF.
getSubtarget<AArch64Subtarget>();
10557 if (
TRI.isGeneralPurposeRegister(MF,
Reg)) {
10570 assert(STI.hasNEON() &&
"Expected to have NEON.");
10576std::optional<DestSourcePair>
10581 if (((
MI.getOpcode() == AArch64::ORRWrs &&
10582 MI.getOperand(1).getReg() == AArch64::WZR &&
10583 MI.getOperand(3).getImm() == 0x0) ||
10584 (
MI.getOpcode() == AArch64::ORRWrr &&
10585 MI.getOperand(1).getReg() == AArch64::WZR)) &&
10587 (!
MI.getOperand(0).getReg().isVirtual() ||
10588 MI.getOperand(0).getSubReg() == 0) &&
10589 (!
MI.getOperand(0).getReg().isPhysical() ||
10594 if (
MI.getOpcode() == AArch64::ORRXrs &&
10595 MI.getOperand(1).getReg() == AArch64::XZR &&
10596 MI.getOperand(3).getImm() == 0x0)
10599 return std::nullopt;
10602std::optional<DestSourcePair>
10604 if ((
MI.getOpcode() == AArch64::ORRWrs &&
10605 MI.getOperand(1).getReg() == AArch64::WZR &&
10606 MI.getOperand(3).getImm() == 0x0) ||
10607 (
MI.getOpcode() == AArch64::ORRWrr &&
10608 MI.getOperand(1).getReg() == AArch64::WZR))
10610 return std::nullopt;
10613std::optional<RegImmPair>
10622 return std::nullopt;
10624 switch (
MI.getOpcode()) {
10626 return std::nullopt;
10627 case AArch64::SUBWri:
10628 case AArch64::SUBXri:
10629 case AArch64::SUBSWri:
10630 case AArch64::SUBSXri:
10633 case AArch64::ADDSWri:
10634 case AArch64::ADDSXri:
10635 case AArch64::ADDWri:
10636 case AArch64::ADDXri: {
10638 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
10639 !
MI.getOperand(2).isImm())
10640 return std::nullopt;
10641 int Shift =
MI.getOperand(3).getImm();
10642 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
10646 return RegImmPair{
MI.getOperand(1).getReg(),
Offset};
10652static std::optional<ParamLoadedValue>
10656 auto DestSrc =
TII->isCopyLikeInstr(
MI);
10658 return std::nullopt;
10660 Register DestReg = DestSrc->Destination->getReg();
10661 Register SrcReg = DestSrc->Source->getReg();
10664 return std::nullopt;
10669 if (DestReg == DescribedReg)
10673 if (
MI.getOpcode() == AArch64::ORRWrs &&
10674 TRI->isSuperRegister(DestReg, DescribedReg))
10678 if (
MI.getOpcode() == AArch64::ORRXrs &&
10679 TRI->isSubRegister(DestReg, DescribedReg)) {
10680 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
10684 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
10685 "Unhandled ORR[XW]rs copy case");
10687 return std::nullopt;
10690bool AArch64InstrInfo::isFunctionSafeToSplit(
const MachineFunction &MF)
const {
10695 if (MF.
getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(
true))
10701bool AArch64InstrInfo::isMBBSafeToSplitToCold(
10705 auto isAsmGoto = [](
const MachineInstr &
MI) {
10706 return MI.getOpcode() == AArch64::INLINEASM_BR;
10716 auto containsMBB = [&
MBB](
const MachineJumpTableEntry &JTE) {
10723 for (
const MachineInstr &
MI :
MBB) {
10724 switch (
MI.getOpcode()) {
10725 case TargetOpcode::G_BRJT:
10726 case AArch64::JumpTableDest32:
10727 case AArch64::JumpTableDest16:
10728 case AArch64::JumpTableDest8:
10739std::optional<ParamLoadedValue>
10742 const MachineFunction *MF =
MI.getMF();
10744 switch (
MI.getOpcode()) {
10745 case AArch64::MOVZWi:
10746 case AArch64::MOVZXi: {
10749 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(),
Reg))
10750 return std::nullopt;
10752 if (!
MI.getOperand(1).isImm())
10753 return std::nullopt;
10754 int64_t Immediate =
MI.getOperand(1).getImm();
10755 int Shift =
MI.getOperand(2).getImm();
10759 case AArch64::ORRWrs:
10760 case AArch64::ORRXrs:
10767bool AArch64InstrInfo::isExtendLikelyToBeFolded(
10770 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
10771 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
10774 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
10778 if (!
MRI.hasOneNonDBGUse(DefReg))
10783 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
10784 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
10787uint64_t AArch64InstrInfo::getElementSizeForOpcode(
unsigned Opc)
const {
10791bool AArch64InstrInfo::isPTestLikeOpcode(
unsigned Opc)
const {
10795bool AArch64InstrInfo::isWhileOpcode(
unsigned Opc)
const {
10800AArch64InstrInfo::getTailDuplicateSize(
CodeGenOptLevel OptLevel)
const {
10804bool AArch64InstrInfo::isLegalAddressingMode(
unsigned NumBytes, int64_t
Offset,
10805 unsigned Scale)
const {
10816 unsigned Shift =
Log2_64(NumBytes);
10817 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
10825 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
10830 return AArch64::BLRNoIP;
10832 return AArch64::BLR;
10837 Register TargetReg,
bool FrameSetup)
const {
10838 assert(TargetReg != AArch64::SP &&
"New top of stack cannot already be in SP");
10850 MF.
insert(MBBInsertPoint, LoopTestMBB);
10853 MF.
insert(MBBInsertPoint, LoopBodyMBB);
10855 MF.
insert(MBBInsertPoint, ExitMBB);
10865 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
10873 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::Bcc))
10879 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
10892 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
10911 MBB.addSuccessor(LoopTestMBB);
10917 return ExitMBB->
begin();
10934 unsigned CompCounterOprNum;
10938 unsigned UpdateCounterOprNum;
10942 bool IsUpdatePriorComp;
10954 TII(MF->getSubtarget().getInstrInfo()),
10955 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
10956 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
10957 CompCounterOprNum(CompCounterOprNum), Update(Update),
10958 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
10959 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
10961 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
10967 std::optional<bool> createTripCountGreaterCondition(
10968 int TC, MachineBasicBlock &
MBB,
10969 SmallVectorImpl<MachineOperand> &CondParam)
override {
10977 void createRemainingIterationsGreaterCondition(
10978 int TC, MachineBasicBlock &
MBB, SmallVectorImpl<MachineOperand> &
Cond,
10979 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts)
override;
10981 void setPreheader(MachineBasicBlock *NewPreheader)
override {}
10983 void adjustTripCount(
int TripCountAdjust)
override {}
10985 bool isMVEExpanderSupported()
override {
return true; }
10998 MBB.getParent()->getSubtarget().getRegisterInfo();
11003 Result =
MRI.createVirtualRegister(
11006 }
else if (
I == ReplaceOprNum) {
11007 MRI.constrainRegClass(ReplaceReg,
11012 MBB.insert(InsertTo, NewMI);
11016void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
11032 assert(CondBranch->getOpcode() == AArch64::Bcc);
11036 if (CondBranch->getOperand(1).getMBB() == LoopBB)
11043 auto AccumulateCond = [&](
Register CurCond,
11045 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
11054 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
11058 for (
int I = 0;
I <= TC; ++
I) {
11064 AccCond = AccumulateCond(AccCond, CC);
11068 if (Update != Comp && IsUpdatePriorComp) {
11070 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
11071 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
11075 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
11077 }
else if (Update != Comp) {
11082 Counter = NextCounter;
11086 if (LastStage0Insts.
empty()) {
11090 if (IsUpdatePriorComp)
11095 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
11098 for (
int I = 0;
I <= TC; ++
I) {
11102 AccCond = AccumulateCond(AccCond, CC);
11103 if (
I != TC && Update != Comp)
11106 Counter = NextCounter;
11122 assert(Phi.getNumOperands() == 5);
11123 if (Phi.getOperand(2).getMBB() ==
MBB) {
11124 RegMBB = Phi.getOperand(1).getReg();
11125 RegOther = Phi.getOperand(3).getReg();
11127 assert(Phi.getOperand(4).getMBB() ==
MBB);
11128 RegMBB = Phi.getOperand(3).getReg();
11129 RegOther = Phi.getOperand(1).getReg();
11134 if (!
Reg.isVirtual())
11137 return MRI.getVRegDef(
Reg)->getParent() != BB;
11143 unsigned &UpdateCounterOprNum,
Register &InitReg,
11144 bool &IsUpdatePriorComp) {
11158 if (!
Reg.isVirtual())
11161 UpdateInst =
nullptr;
11162 UpdateCounterOprNum = 0;
11164 IsUpdatePriorComp =
true;
11168 if (Def->getParent() != LoopBB)
11170 if (Def->isCopy()) {
11172 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
11174 CurReg = Def->getOperand(1).getReg();
11175 }
else if (Def->isPHI()) {
11179 IsUpdatePriorComp =
false;
11184 switch (Def->getOpcode()) {
11185 case AArch64::ADDSXri:
11186 case AArch64::ADDSWri:
11187 case AArch64::SUBSXri:
11188 case AArch64::SUBSWri:
11189 case AArch64::ADDXri:
11190 case AArch64::ADDWri:
11191 case AArch64::SUBXri:
11192 case AArch64::SUBWri:
11194 UpdateCounterOprNum = 1;
11196 case AArch64::ADDSXrr:
11197 case AArch64::ADDSWrr:
11198 case AArch64::SUBSXrr:
11199 case AArch64::SUBSWrr:
11200 case AArch64::ADDXrr:
11201 case AArch64::ADDWrr:
11202 case AArch64::SUBXrr:
11203 case AArch64::SUBWrr:
11206 UpdateCounterOprNum = 1;
11208 UpdateCounterOprNum = 2;
11215 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
11230std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
11241 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
11252 if (
TBB == LoopBB && FBB == LoopBB)
11256 if (
TBB != LoopBB && FBB ==
nullptr)
11259 assert((
TBB == LoopBB || FBB == LoopBB) &&
11260 "The Loop must be a single-basic-block loop");
11265 if (CondBranch->
getOpcode() != AArch64::Bcc)
11273 unsigned CompCounterOprNum = 0;
11275 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
11279 switch (
MI.getOpcode()) {
11280 case AArch64::SUBSXri:
11281 case AArch64::SUBSWri:
11282 case AArch64::ADDSXri:
11283 case AArch64::ADDSWri:
11285 CompCounterOprNum = 1;
11287 case AArch64::ADDSWrr:
11288 case AArch64::ADDSXrr:
11289 case AArch64::SUBSWrr:
11290 case AArch64::SUBSXrr:
11294 if (isWhileOpcode(
MI.getOpcode())) {
11301 if (CompCounterOprNum == 0) {
11303 CompCounterOprNum = 2;
11305 CompCounterOprNum = 1;
11317 bool IsUpdatePriorComp;
11318 unsigned UpdateCounterOprNum;
11320 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
11323 return std::make_unique<AArch64PipelinerLoopInfo>(
11324 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
11334 TypeSize Scale(0U,
false), Width(0U,
false);
11335 int64_t MinOffset, MaxOffset;
11336 if (
getMemOpInfo(
MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
11338 if (
MI.getOperand(ImmIdx).isImm() && !
MI.getOperand(ImmIdx - 1).isFI()) {
11339 int64_t Imm =
MI.getOperand(ImmIdx).getImm();
11340 if (Imm < MinOffset || Imm > MaxOffset) {
11341 ErrInfo =
"Unexpected immediate on load/store instruction";
11347 const MCInstrDesc &MCID =
MI.getDesc();
11349 const MachineOperand &MO =
MI.getOperand(
Op);
11353 ErrInfo =
"OPERAND_IMPLICIT_IMM_0 should be 0";
11362 ErrInfo =
"OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
11373#define GET_INSTRINFO_HELPERS
11374#define GET_INSTRMAP_INFO
11375#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static cl::opt< unsigned > GatherOptSearchLimit("aarch64-search-limit", cl::Hidden, cl::init(2048), cl::desc("Restrict range of instructions to search for the " "machine-combiner gather pattern optimization"))
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment, StringRef RegScale={})
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, MCRegister Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static bool isPostIndexLdStOpcode(unsigned Opcode)
Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
static unsigned getBranchDisplacementBits(unsigned Opc)
static cl::opt< unsigned > CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(9), cl::desc("Restrict range of CB instructions (DEBUG)"))
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static void appendReadRegExpr(SmallVectorImpl< char > &Expr, unsigned RegNum)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool getLoadPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Search for patterns of LD instructions we can optimize.
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static void appendLoadRegExpr(SmallVectorImpl< char > &Expr, int64_t OffsetFromDefCFA)
static void appendConstantExpr(SmallVectorImpl< char > &Expr, int64_t Constant, dwarf::LocationAtom Operation)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static void generateGatherLanePattern(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned Pattern, unsigned NumLanes)
Generate optimized instruction sequence for gather load patterns to improve Memory-Level Parallelism ...
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool getGatherLanePattern(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, unsigned LoadLaneOpCode, unsigned NumLanes)
Check if the given instruction forms a gather load pattern that can be optimized for better Memory-Le...
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
Machine Check Debug Module
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
MachineInstr unsigned OpIdx
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallSet class.
This file defines the SmallVector class.
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setOutliningStyle(const std::string &Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isAsCheapAsAMove(const MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This is an important base class in LLVM.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
Instructions::iterator instr_iterator
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
LLVM_ABI bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
LLVM_ABI uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
LLVM_ABI bool isLoadFoldBarrier() const
Returns true if it is illegal to fold a load across this instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
LLVM_ABI void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
LLVM_ABI void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
LLVM_ABI MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
LLVM_ABI MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Value * getOperand(unsigned i) const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_ARM64EC_CALLMANGLE
MO_ARM64EC_CALLMANGLE - Operand refers to the Arm64EC-mangled version of a symbol,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ ScalablePredicateVector
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
LLVM_ABI Instruction & back() const
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool succeeded(LogicalResult Result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
static bool isIndirectBranchOpcode(int Opc)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
static bool isSEHInstruction(const MachineInstr &MI)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
unsigned getUndefRegState(bool B)
static MCRegister getXRegFromWReg(MCRegister Reg)
unsigned getDefRegState(bool B)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA, std::optional< int64_t > IncomingVGOffsetFromDefCFA)
unsigned getKillRegState(bool B)
FunctionAddr VTableAddr Next
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
static bool isUncondBranchOpcode(int Opc)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
void appendLEB128(SmallVectorImpl< U > &Buffer, T Value)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
static const MachineMemOperand::Flags MOStridedAccess
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
LLVM_ABI static const MBBSectionID ColdSectionID
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.