62#define GET_INSTRINFO_CTOR_DTOR
63#include "AArch64GenInstrInfo.inc"
67 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
71 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
75 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
79 cl::desc(
"Restrict range of B instructions (DEBUG)"));
84 RI(STI.getTargetTriple()), Subtarget(STI) {}
95 auto Op =
MI.getOpcode();
96 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
97 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
101 if (
MI.isMetaInstruction())
106 unsigned NumBytes = 0;
110 NumBytes =
Desc.getSize() ?
Desc.getSize() : 4;
113 if (!MFI->shouldSignReturnAddress(MF))
125 switch (
Desc.getOpcode()) {
128 return Desc.getSize();
135 case TargetOpcode::STACKMAP:
138 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
140 case TargetOpcode::PATCHPOINT:
143 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
145 case TargetOpcode::STATEPOINT:
147 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
152 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
157 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
159 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
160 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
164 case TargetOpcode::PATCHABLE_EVENT_CALL:
170 NumBytes =
MI.getOperand(1).getImm();
172 case TargetOpcode::BUNDLE:
173 NumBytes = getInstBundleLength(
MI);
180unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
184 while (++
I != E &&
I->isInsideBundle()) {
185 assert(!
I->isBundle() &&
"No nested bundle!");
244 int64_t BrOffset)
const {
246 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
247 "over conditional branch expansion");
248 return isIntN(Bits, BrOffset / 4);
253 switch (
MI.getOpcode()) {
257 return MI.getOperand(0).getMBB();
262 return MI.getOperand(2).getMBB();
268 return MI.getOperand(1).getMBB();
278 assert(RS &&
"RegScavenger required for long branching");
280 "new block should be inserted for expanding unconditional branch");
283 "restore block should be inserted for restoring clobbered registers");
288 if (!isInt<33>(BrOffset))
290 "Branch offsets outside of the signed 33-bit range not supported");
304 constexpr Register Reg = AArch64::X16;
306 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
314 if (Scavenged != AArch64::NoRegister &&
316 buildIndirectBranch(Scavenged, NewDestBB);
326 "Unable to insert indirect branch inside function that has red zone");
349 bool AllowModify)
const {
356 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
357 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
361 if (!isUnpredicatedTerminator(*
I))
368 unsigned LastOpc = LastInst->
getOpcode();
369 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
384 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
391 LastInst = SecondLastInst;
393 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
398 SecondLastInst = &*
I;
399 SecondLastOpc = SecondLastInst->
getOpcode();
410 LastInst = SecondLastInst;
412 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
414 "unreachable unconditional branches removed above");
423 SecondLastInst = &*
I;
424 SecondLastOpc = SecondLastInst->
getOpcode();
428 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
444 I->eraseFromParent();
453 I->eraseFromParent();
462 MachineBranchPredicate &MBP,
463 bool AllowModify)
const {
473 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
474 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
478 if (!isUnpredicatedTerminator(*
I))
483 unsigned LastOpc = LastInst->
getOpcode();
498 assert(MBP.TrueDest &&
"expected!");
501 MBP.ConditionDef =
nullptr;
502 MBP.SingleUseCondition =
false;
506 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
507 : MachineBranchPredicate::PRED_EQ;
513 if (
Cond[0].getImm() != -1) {
519 switch (
Cond[1].getImm()) {
523 Cond[1].setImm(AArch64::CBNZW);
526 Cond[1].setImm(AArch64::CBZW);
529 Cond[1].setImm(AArch64::CBNZX);
532 Cond[1].setImm(AArch64::CBZX);
535 Cond[1].setImm(AArch64::TBNZW);
538 Cond[1].setImm(AArch64::TBZW);
541 Cond[1].setImm(AArch64::TBNZX);
544 Cond[1].setImm(AArch64::TBZX);
553 int *BytesRemoved)
const {
563 I->eraseFromParent();
580 I->eraseFromParent();
587void AArch64InstrInfo::instantiateCondBranch(
590 if (
Cond[0].getImm() != -1) {
608 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
647 unsigned *NewVReg =
nullptr) {
652 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
655 unsigned SrcOpNum = 0;
657 case AArch64::ADDSXri:
658 case AArch64::ADDSWri:
665 case AArch64::ADDXri:
666 case AArch64::ADDWri:
672 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
675 case AArch64::ORNXrr:
676 case AArch64::ORNWrr: {
679 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
682 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
686 case AArch64::SUBSXrr:
687 case AArch64::SUBSWrr:
694 case AArch64::SUBXrr:
695 case AArch64::SUBWrr: {
698 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
701 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
707 assert(Opc && SrcOpNum &&
"Missing parameters");
719 int &FalseCycles)
const {
723 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
730 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
734 unsigned ExtraCondLat =
Cond.size() != 1;
738 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
739 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
741 CondCycles = 1 + ExtraCondLat;
742 TrueCycles = FalseCycles = 1;
752 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
753 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
754 CondCycles = 5 + ExtraCondLat;
755 TrueCycles = FalseCycles = 2;
772 switch (
Cond.size()) {
781 switch (
Cond[1].getImm()) {
804 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
810 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
820 switch (
Cond[1].getImm()) {
833 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
849 bool TryFold =
false;
850 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
851 RC = &AArch64::GPR64RegClass;
852 Opc = AArch64::CSELXr;
854 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
855 RC = &AArch64::GPR32RegClass;
856 Opc = AArch64::CSELWr;
858 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
859 RC = &AArch64::FPR64RegClass;
860 Opc = AArch64::FCSELDrrr;
861 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
862 RC = &AArch64::FPR32RegClass;
863 Opc = AArch64::FCSELSrrr;
865 assert(RC &&
"Unsupported regclass");
869 unsigned NewVReg = 0;
884 MRI.clearKillFlags(NewVReg);
889 MRI.constrainRegClass(TrueReg, RC);
890 MRI.constrainRegClass(FalseReg, RC);
905 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
910 return Is.
size() <= 2;
916 if (Subtarget.hasExynosCheapAsMoveHandling()) {
917 if (isExynosCheapAsMove(
MI))
919 return MI.isAsCheapAsAMove();
922 switch (
MI.getOpcode()) {
924 return MI.isAsCheapAsAMove();
926 case AArch64::ADDWrs:
927 case AArch64::ADDXrs:
928 case AArch64::SUBWrs:
929 case AArch64::SUBXrs:
930 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
935 case AArch64::MOVi32imm:
937 case AArch64::MOVi64imm:
943 switch (
MI.getOpcode()) {
947 case AArch64::ADDWrs:
948 case AArch64::ADDXrs:
949 case AArch64::ADDSWrs:
950 case AArch64::ADDSXrs: {
951 unsigned Imm =
MI.getOperand(3).getImm();
958 case AArch64::ADDWrx:
959 case AArch64::ADDXrx:
960 case AArch64::ADDXrx64:
961 case AArch64::ADDSWrx:
962 case AArch64::ADDSXrx:
963 case AArch64::ADDSXrx64: {
964 unsigned Imm =
MI.getOperand(3).getImm();
976 case AArch64::SUBWrs:
977 case AArch64::SUBSWrs: {
978 unsigned Imm =
MI.getOperand(3).getImm();
980 return ShiftVal == 0 ||
984 case AArch64::SUBXrs:
985 case AArch64::SUBSXrs: {
986 unsigned Imm =
MI.getOperand(3).getImm();
988 return ShiftVal == 0 ||
992 case AArch64::SUBWrx:
993 case AArch64::SUBXrx:
994 case AArch64::SUBXrx64:
995 case AArch64::SUBSWrx:
996 case AArch64::SUBSXrx:
997 case AArch64::SUBSXrx64: {
998 unsigned Imm =
MI.getOperand(3).getImm();
1010 case AArch64::LDRBBroW:
1011 case AArch64::LDRBBroX:
1012 case AArch64::LDRBroW:
1013 case AArch64::LDRBroX:
1014 case AArch64::LDRDroW:
1015 case AArch64::LDRDroX:
1016 case AArch64::LDRHHroW:
1017 case AArch64::LDRHHroX:
1018 case AArch64::LDRHroW:
1019 case AArch64::LDRHroX:
1020 case AArch64::LDRQroW:
1021 case AArch64::LDRQroX:
1022 case AArch64::LDRSBWroW:
1023 case AArch64::LDRSBWroX:
1024 case AArch64::LDRSBXroW:
1025 case AArch64::LDRSBXroX:
1026 case AArch64::LDRSHWroW:
1027 case AArch64::LDRSHWroX:
1028 case AArch64::LDRSHXroW:
1029 case AArch64::LDRSHXroX:
1030 case AArch64::LDRSWroW:
1031 case AArch64::LDRSWroX:
1032 case AArch64::LDRSroW:
1033 case AArch64::LDRSroX:
1034 case AArch64::LDRWroW:
1035 case AArch64::LDRWroX:
1036 case AArch64::LDRXroW:
1037 case AArch64::LDRXroX:
1038 case AArch64::PRFMroW:
1039 case AArch64::PRFMroX:
1040 case AArch64::STRBBroW:
1041 case AArch64::STRBBroX:
1042 case AArch64::STRBroW:
1043 case AArch64::STRBroX:
1044 case AArch64::STRDroW:
1045 case AArch64::STRDroX:
1046 case AArch64::STRHHroW:
1047 case AArch64::STRHHroX:
1048 case AArch64::STRHroW:
1049 case AArch64::STRHroX:
1050 case AArch64::STRQroW:
1051 case AArch64::STRQroX:
1052 case AArch64::STRSroW:
1053 case AArch64::STRSroX:
1054 case AArch64::STRWroW:
1055 case AArch64::STRWroX:
1056 case AArch64::STRXroW:
1057 case AArch64::STRXroX: {
1058 unsigned IsSigned =
MI.getOperand(3).getImm();
1065 unsigned Opc =
MI.getOpcode();
1069 case AArch64::SEH_StackAlloc:
1070 case AArch64::SEH_SaveFPLR:
1071 case AArch64::SEH_SaveFPLR_X:
1072 case AArch64::SEH_SaveReg:
1073 case AArch64::SEH_SaveReg_X:
1074 case AArch64::SEH_SaveRegP:
1075 case AArch64::SEH_SaveRegP_X:
1076 case AArch64::SEH_SaveFReg:
1077 case AArch64::SEH_SaveFReg_X:
1078 case AArch64::SEH_SaveFRegP:
1079 case AArch64::SEH_SaveFRegP_X:
1080 case AArch64::SEH_SetFP:
1081 case AArch64::SEH_AddFP:
1082 case AArch64::SEH_Nop:
1083 case AArch64::SEH_PrologEnd:
1084 case AArch64::SEH_EpilogStart:
1085 case AArch64::SEH_EpilogEnd:
1086 case AArch64::SEH_PACSignLR:
1087 case AArch64::SEH_SaveAnyRegQP:
1088 case AArch64::SEH_SaveAnyRegQPX:
1095 unsigned &SubIdx)
const {
1096 switch (
MI.getOpcode()) {
1099 case AArch64::SBFMXri:
1100 case AArch64::UBFMXri:
1103 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1106 SrcReg =
MI.getOperand(1).getReg();
1107 DstReg =
MI.getOperand(0).getReg();
1108 SubIdx = AArch64::sub_32;
1117 int64_t OffsetA = 0, OffsetB = 0;
1118 TypeSize WidthA(0,
false), WidthB(0,
false);
1119 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1140 OffsetAIsScalable == OffsetBIsScalable) {
1141 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1142 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1143 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1144 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1162 switch (
MI.getOpcode()) {
1165 if (
MI.getOperand(0).getImm() == 0x14)
1172 case AArch64::MSRpstatesvcrImm1:
1179 auto Next = std::next(
MI.getIterator());
1180 return Next !=
MBB->
end() && Next->isCFIInstruction();
1187 Register &SrcReg2, int64_t &CmpMask,
1188 int64_t &CmpValue)
const {
1191 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1192 if (!
MI.getOperand(1).isReg())
1195 switch (
MI.getOpcode()) {
1198 case AArch64::PTEST_PP:
1199 case AArch64::PTEST_PP_ANY:
1200 SrcReg =
MI.getOperand(0).getReg();
1201 SrcReg2 =
MI.getOperand(1).getReg();
1206 case AArch64::SUBSWrr:
1207 case AArch64::SUBSWrs:
1208 case AArch64::SUBSWrx:
1209 case AArch64::SUBSXrr:
1210 case AArch64::SUBSXrs:
1211 case AArch64::SUBSXrx:
1212 case AArch64::ADDSWrr:
1213 case AArch64::ADDSWrs:
1214 case AArch64::ADDSWrx:
1215 case AArch64::ADDSXrr:
1216 case AArch64::ADDSXrs:
1217 case AArch64::ADDSXrx:
1219 SrcReg =
MI.getOperand(1).getReg();
1220 SrcReg2 =
MI.getOperand(2).getReg();
1224 case AArch64::SUBSWri:
1225 case AArch64::ADDSWri:
1226 case AArch64::SUBSXri:
1227 case AArch64::ADDSXri:
1228 SrcReg =
MI.getOperand(1).getReg();
1231 CmpValue =
MI.getOperand(2).getImm();
1233 case AArch64::ANDSWri:
1234 case AArch64::ANDSXri:
1237 SrcReg =
MI.getOperand(1).getReg();
1241 MI.getOperand(2).getImm(),
1242 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1251 assert(
MBB &&
"Can't get MachineBasicBlock here");
1253 assert(MF &&
"Can't get MachineFunction here");
1258 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1262 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1265 if (!OpRegCstraints)
1273 "Operand has register constraints without being a register!");
1276 if (Reg.isPhysical()) {
1277 if (!OpRegCstraints->
contains(Reg))
1280 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1293 bool MIDefinesZeroReg =
false;
1294 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1295 MI.definesRegister(AArch64::XZR,
nullptr))
1296 MIDefinesZeroReg =
true;
1298 switch (
MI.getOpcode()) {
1300 return MI.getOpcode();
1301 case AArch64::ADDSWrr:
1302 return AArch64::ADDWrr;
1303 case AArch64::ADDSWri:
1304 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1305 case AArch64::ADDSWrs:
1306 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1307 case AArch64::ADDSWrx:
1308 return AArch64::ADDWrx;
1309 case AArch64::ADDSXrr:
1310 return AArch64::ADDXrr;
1311 case AArch64::ADDSXri:
1312 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1313 case AArch64::ADDSXrs:
1314 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1315 case AArch64::ADDSXrx:
1316 return AArch64::ADDXrx;
1317 case AArch64::SUBSWrr:
1318 return AArch64::SUBWrr;
1319 case AArch64::SUBSWri:
1320 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1321 case AArch64::SUBSWrs:
1322 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1323 case AArch64::SUBSWrx:
1324 return AArch64::SUBWrx;
1325 case AArch64::SUBSXrr:
1326 return AArch64::SUBXrr;
1327 case AArch64::SUBSXri:
1328 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1329 case AArch64::SUBSXrs:
1330 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1331 case AArch64::SUBSXrx:
1332 return AArch64::SUBXrx;
1347 if (To == To->getParent()->begin())
1352 if (To->getParent() !=
From->getParent())
1364 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1365 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1371std::optional<unsigned>
1375 unsigned MaskOpcode =
Mask->getOpcode();
1376 unsigned PredOpcode = Pred->
getOpcode();
1380 if (PredIsWhileLike) {
1384 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1398 if (PredIsPTestLike) {
1403 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1413 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1442 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1450 switch (PredOpcode) {
1451 case AArch64::AND_PPzPP:
1452 case AArch64::BIC_PPzPP:
1453 case AArch64::EOR_PPzPP:
1454 case AArch64::NAND_PPzPP:
1455 case AArch64::NOR_PPzPP:
1456 case AArch64::ORN_PPzPP:
1457 case AArch64::ORR_PPzPP:
1458 case AArch64::BRKA_PPzP:
1459 case AArch64::BRKPA_PPzPP:
1460 case AArch64::BRKB_PPzP:
1461 case AArch64::BRKPB_PPzPP:
1462 case AArch64::RDFFR_PPz: {
1466 if (Mask != PredMask)
1470 case AArch64::BRKN_PPzP: {
1474 if ((MaskOpcode != AArch64::PTRUE_B) ||
1475 (
Mask->getOperand(1).getImm() != 31))
1479 case AArch64::PTRUE_B:
1492bool AArch64InstrInfo::optimizePTestInstr(
1493 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1495 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1496 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1497 unsigned PredOpcode = Pred->
getOpcode();
1498 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1514 if (*NewOp != PredOpcode) {
1525 for (; i !=
e; ++i) {
1556 if (DeadNZCVIdx != -1) {
1575 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1576 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1577 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1586 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1588 return (CmpValue == 0 || CmpValue == 1) &&
1589 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1597 switch (Instr.getOpcode()) {
1599 return AArch64::INSTRUCTION_LIST_END;
1601 case AArch64::ADDSWrr:
1602 case AArch64::ADDSWri:
1603 case AArch64::ADDSXrr:
1604 case AArch64::ADDSXri:
1605 case AArch64::SUBSWrr:
1606 case AArch64::SUBSWri:
1607 case AArch64::SUBSXrr:
1608 case AArch64::SUBSXri:
1609 return Instr.getOpcode();
1611 case AArch64::ADDWrr:
1612 return AArch64::ADDSWrr;
1613 case AArch64::ADDWri:
1614 return AArch64::ADDSWri;
1615 case AArch64::ADDXrr:
1616 return AArch64::ADDSXrr;
1617 case AArch64::ADDXri:
1618 return AArch64::ADDSXri;
1619 case AArch64::ADCWr:
1620 return AArch64::ADCSWr;
1621 case AArch64::ADCXr:
1622 return AArch64::ADCSXr;
1623 case AArch64::SUBWrr:
1624 return AArch64::SUBSWrr;
1625 case AArch64::SUBWri:
1626 return AArch64::SUBSWri;
1627 case AArch64::SUBXrr:
1628 return AArch64::SUBSXrr;
1629 case AArch64::SUBXri:
1630 return AArch64::SUBSXri;
1631 case AArch64::SBCWr:
1632 return AArch64::SBCSWr;
1633 case AArch64::SBCXr:
1634 return AArch64::SBCSXr;
1635 case AArch64::ANDWri:
1636 return AArch64::ANDSWri;
1637 case AArch64::ANDXri:
1638 return AArch64::ANDSXri;
1645 if (BB->isLiveIn(AArch64::NZCV))
1654 switch (Instr.getOpcode()) {
1658 case AArch64::Bcc: {
1659 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1664 case AArch64::CSINVWr:
1665 case AArch64::CSINVXr:
1666 case AArch64::CSINCWr:
1667 case AArch64::CSINCXr:
1668 case AArch64::CSELWr:
1669 case AArch64::CSELXr:
1670 case AArch64::CSNEGWr:
1671 case AArch64::CSNEGXr:
1672 case AArch64::FCSELSrrr:
1673 case AArch64::FCSELDrrr: {
1674 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1687 Instr.getOperand(CCIdx).getImm())
1740std::optional<UsedNZCV>
1745 if (
MI.getParent() != CmpParent)
1746 return std::nullopt;
1749 return std::nullopt;
1754 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1757 return std::nullopt;
1762 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1765 return NZCVUsedAfterCmp;
1769 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1773 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1795 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1801 "Caller guarantees that CmpInstr compares with constant 0");
1804 if (!NZVCUsed || NZVCUsed->C)
1826bool AArch64InstrInfo::substituteCmpToZero(
1837 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1844 MI->setDesc(
get(NewOpc));
1849 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1861 assert((CmpValue == 0 || CmpValue == 1) &&
1862 "Only comparisons to 0 or 1 considered for removal!");
1865 unsigned MIOpc =
MI.getOpcode();
1866 if (MIOpc == AArch64::CSINCWr) {
1867 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1868 MI.getOperand(2).getReg() != AArch64::WZR)
1870 }
else if (MIOpc == AArch64::CSINCXr) {
1871 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1872 MI.getOperand(2).getReg() != AArch64::XZR)
1882 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1886 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1888 if (CmpValue && !IsSubsRegImm)
1890 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1895 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1898 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1902 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1905 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1906 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1909 if (MIUsedNZCV.
N && !CmpValue)
1951bool AArch64InstrInfo::removeCmpToZeroOrOne(
1959 bool IsInvertCC =
false;
1969 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1980 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1981 MI.getOpcode() != AArch64::CATCHRET)
1989 if (
MI.getOpcode() == AArch64::CATCHRET) {
1998 FirstEpilogSEH = std::prev(FirstEpilogSEH);
2000 FirstEpilogSEH = std::next(FirstEpilogSEH);
2015 if (M.getStackProtectorGuard() ==
"sysreg") {
2025 int Offset = M.getStackProtectorGuardOffset();
2077 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2086 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2128 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2151 switch (
MI.getOpcode()) {
2154 case AArch64::MOVZWi:
2155 case AArch64::MOVZXi:
2156 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2157 assert(
MI.getDesc().getNumOperands() == 3 &&
2158 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2162 case AArch64::ANDWri:
2163 return MI.getOperand(1).getReg() == AArch64::WZR;
2164 case AArch64::ANDXri:
2165 return MI.getOperand(1).getReg() == AArch64::XZR;
2166 case TargetOpcode::COPY:
2167 return MI.getOperand(1).getReg() == AArch64::WZR;
2175 switch (
MI.getOpcode()) {
2178 case TargetOpcode::COPY: {
2181 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2182 AArch64::GPR64RegClass.
contains(DstReg));
2184 case AArch64::ORRXrs:
2185 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2186 assert(
MI.getDesc().getNumOperands() == 4 &&
2187 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2191 case AArch64::ADDXri:
2192 if (
MI.getOperand(2).getImm() == 0) {
2193 assert(
MI.getDesc().getNumOperands() == 4 &&
2194 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2205 switch (
MI.getOpcode()) {
2208 case TargetOpcode::COPY: {
2210 return AArch64::FPR128RegClass.contains(DstReg);
2212 case AArch64::ORRv16i8:
2213 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2214 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2215 "invalid ORRv16i8 operands");
2224 int &FrameIndex)
const {
2225 switch (
MI.getOpcode()) {
2228 case AArch64::LDRWui:
2229 case AArch64::LDRXui:
2230 case AArch64::LDRBui:
2231 case AArch64::LDRHui:
2232 case AArch64::LDRSui:
2233 case AArch64::LDRDui:
2234 case AArch64::LDRQui:
2235 case AArch64::LDR_PXI:
2236 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2237 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2238 FrameIndex =
MI.getOperand(1).getIndex();
2239 return MI.getOperand(0).getReg();
2248 int &FrameIndex)
const {
2249 switch (
MI.getOpcode()) {
2252 case AArch64::STRWui:
2253 case AArch64::STRXui:
2254 case AArch64::STRBui:
2255 case AArch64::STRHui:
2256 case AArch64::STRSui:
2257 case AArch64::STRDui:
2258 case AArch64::STRQui:
2259 case AArch64::STR_PXI:
2260 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2261 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2262 FrameIndex =
MI.getOperand(1).getIndex();
2263 return MI.getOperand(0).getReg();
2273 return MMO->getFlags() & MOSuppressPair;
2279 if (
MI.memoperands_empty())
2287 return MMO->getFlags() & MOStridedAccess;
2295 case AArch64::STURSi:
2296 case AArch64::STRSpre:
2297 case AArch64::STURDi:
2298 case AArch64::STRDpre:
2299 case AArch64::STURQi:
2300 case AArch64::STRQpre:
2301 case AArch64::STURBBi:
2302 case AArch64::STURHHi:
2303 case AArch64::STURWi:
2304 case AArch64::STRWpre:
2305 case AArch64::STURXi:
2306 case AArch64::STRXpre:
2307 case AArch64::LDURSi:
2308 case AArch64::LDRSpre:
2309 case AArch64::LDURDi:
2310 case AArch64::LDRDpre:
2311 case AArch64::LDURQi:
2312 case AArch64::LDRQpre:
2313 case AArch64::LDURWi:
2314 case AArch64::LDRWpre:
2315 case AArch64::LDURXi:
2316 case AArch64::LDRXpre:
2317 case AArch64::LDRSWpre:
2318 case AArch64::LDURSWi:
2319 case AArch64::LDURHHi:
2320 case AArch64::LDURBBi:
2321 case AArch64::LDURSBWi:
2322 case AArch64::LDURSHWi:
2330 case AArch64::PRFMui:
return AArch64::PRFUMi;
2331 case AArch64::LDRXui:
return AArch64::LDURXi;
2332 case AArch64::LDRWui:
return AArch64::LDURWi;
2333 case AArch64::LDRBui:
return AArch64::LDURBi;
2334 case AArch64::LDRHui:
return AArch64::LDURHi;
2335 case AArch64::LDRSui:
return AArch64::LDURSi;
2336 case AArch64::LDRDui:
return AArch64::LDURDi;
2337 case AArch64::LDRQui:
return AArch64::LDURQi;
2338 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2339 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2340 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2341 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2342 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2343 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2344 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2345 case AArch64::STRXui:
return AArch64::STURXi;
2346 case AArch64::STRWui:
return AArch64::STURWi;
2347 case AArch64::STRBui:
return AArch64::STURBi;
2348 case AArch64::STRHui:
return AArch64::STURHi;
2349 case AArch64::STRSui:
return AArch64::STURSi;
2350 case AArch64::STRDui:
return AArch64::STURDi;
2351 case AArch64::STRQui:
return AArch64::STURQi;
2352 case AArch64::STRBBui:
return AArch64::STURBBi;
2353 case AArch64::STRHHui:
return AArch64::STURHHi;
2362 case AArch64::LDAPURBi:
2363 case AArch64::LDAPURHi:
2364 case AArch64::LDAPURi:
2365 case AArch64::LDAPURSBWi:
2366 case AArch64::LDAPURSBXi:
2367 case AArch64::LDAPURSHWi:
2368 case AArch64::LDAPURSHXi:
2369 case AArch64::LDAPURSWi:
2370 case AArch64::LDAPURXi:
2371 case AArch64::LDR_PPXI:
2372 case AArch64::LDR_PXI:
2373 case AArch64::LDR_ZXI:
2374 case AArch64::LDR_ZZXI:
2375 case AArch64::LDR_ZZZXI:
2376 case AArch64::LDR_ZZZZXI:
2377 case AArch64::LDRBBui:
2378 case AArch64::LDRBui:
2379 case AArch64::LDRDui:
2380 case AArch64::LDRHHui:
2381 case AArch64::LDRHui:
2382 case AArch64::LDRQui:
2383 case AArch64::LDRSBWui:
2384 case AArch64::LDRSBXui:
2385 case AArch64::LDRSHWui:
2386 case AArch64::LDRSHXui:
2387 case AArch64::LDRSui:
2388 case AArch64::LDRSWui:
2389 case AArch64::LDRWui:
2390 case AArch64::LDRXui:
2391 case AArch64::LDURBBi:
2392 case AArch64::LDURBi:
2393 case AArch64::LDURDi:
2394 case AArch64::LDURHHi:
2395 case AArch64::LDURHi:
2396 case AArch64::LDURQi:
2397 case AArch64::LDURSBWi:
2398 case AArch64::LDURSBXi:
2399 case AArch64::LDURSHWi:
2400 case AArch64::LDURSHXi:
2401 case AArch64::LDURSi:
2402 case AArch64::LDURSWi:
2403 case AArch64::LDURWi:
2404 case AArch64::LDURXi:
2405 case AArch64::PRFMui:
2406 case AArch64::PRFUMi:
2407 case AArch64::ST2Gi:
2409 case AArch64::STLURBi:
2410 case AArch64::STLURHi:
2411 case AArch64::STLURWi:
2412 case AArch64::STLURXi:
2413 case AArch64::StoreSwiftAsyncContext:
2414 case AArch64::STR_PPXI:
2415 case AArch64::STR_PXI:
2416 case AArch64::STR_ZXI:
2417 case AArch64::STR_ZZXI:
2418 case AArch64::STR_ZZZXI:
2419 case AArch64::STR_ZZZZXI:
2420 case AArch64::STRBBui:
2421 case AArch64::STRBui:
2422 case AArch64::STRDui:
2423 case AArch64::STRHHui:
2424 case AArch64::STRHui:
2425 case AArch64::STRQui:
2426 case AArch64::STRSui:
2427 case AArch64::STRWui:
2428 case AArch64::STRXui:
2429 case AArch64::STURBBi:
2430 case AArch64::STURBi:
2431 case AArch64::STURDi:
2432 case AArch64::STURHHi:
2433 case AArch64::STURHi:
2434 case AArch64::STURQi:
2435 case AArch64::STURSi:
2436 case AArch64::STURWi:
2437 case AArch64::STURXi:
2438 case AArch64::STZ2Gi:
2439 case AArch64::STZGi:
2440 case AArch64::TAGPstack:
2442 case AArch64::LD1B_D_IMM:
2443 case AArch64::LD1B_H_IMM:
2444 case AArch64::LD1B_IMM:
2445 case AArch64::LD1B_S_IMM:
2446 case AArch64::LD1D_IMM:
2447 case AArch64::LD1H_D_IMM:
2448 case AArch64::LD1H_IMM:
2449 case AArch64::LD1H_S_IMM:
2450 case AArch64::LD1RB_D_IMM:
2451 case AArch64::LD1RB_H_IMM:
2452 case AArch64::LD1RB_IMM:
2453 case AArch64::LD1RB_S_IMM:
2454 case AArch64::LD1RD_IMM:
2455 case AArch64::LD1RH_D_IMM:
2456 case AArch64::LD1RH_IMM:
2457 case AArch64::LD1RH_S_IMM:
2458 case AArch64::LD1RSB_D_IMM:
2459 case AArch64::LD1RSB_H_IMM:
2460 case AArch64::LD1RSB_S_IMM:
2461 case AArch64::LD1RSH_D_IMM:
2462 case AArch64::LD1RSH_S_IMM:
2463 case AArch64::LD1RSW_IMM:
2464 case AArch64::LD1RW_D_IMM:
2465 case AArch64::LD1RW_IMM:
2466 case AArch64::LD1SB_D_IMM:
2467 case AArch64::LD1SB_H_IMM:
2468 case AArch64::LD1SB_S_IMM:
2469 case AArch64::LD1SH_D_IMM:
2470 case AArch64::LD1SH_S_IMM:
2471 case AArch64::LD1SW_D_IMM:
2472 case AArch64::LD1W_D_IMM:
2473 case AArch64::LD1W_IMM:
2474 case AArch64::LD2B_IMM:
2475 case AArch64::LD2D_IMM:
2476 case AArch64::LD2H_IMM:
2477 case AArch64::LD2W_IMM:
2478 case AArch64::LD3B_IMM:
2479 case AArch64::LD3D_IMM:
2480 case AArch64::LD3H_IMM:
2481 case AArch64::LD3W_IMM:
2482 case AArch64::LD4B_IMM:
2483 case AArch64::LD4D_IMM:
2484 case AArch64::LD4H_IMM:
2485 case AArch64::LD4W_IMM:
2487 case AArch64::LDNF1B_D_IMM:
2488 case AArch64::LDNF1B_H_IMM:
2489 case AArch64::LDNF1B_IMM:
2490 case AArch64::LDNF1B_S_IMM:
2491 case AArch64::LDNF1D_IMM:
2492 case AArch64::LDNF1H_D_IMM:
2493 case AArch64::LDNF1H_IMM:
2494 case AArch64::LDNF1H_S_IMM:
2495 case AArch64::LDNF1SB_D_IMM:
2496 case AArch64::LDNF1SB_H_IMM:
2497 case AArch64::LDNF1SB_S_IMM:
2498 case AArch64::LDNF1SH_D_IMM:
2499 case AArch64::LDNF1SH_S_IMM:
2500 case AArch64::LDNF1SW_D_IMM:
2501 case AArch64::LDNF1W_D_IMM:
2502 case AArch64::LDNF1W_IMM:
2503 case AArch64::LDNPDi:
2504 case AArch64::LDNPQi:
2505 case AArch64::LDNPSi:
2506 case AArch64::LDNPWi:
2507 case AArch64::LDNPXi:
2508 case AArch64::LDNT1B_ZRI:
2509 case AArch64::LDNT1D_ZRI:
2510 case AArch64::LDNT1H_ZRI:
2511 case AArch64::LDNT1W_ZRI:
2512 case AArch64::LDPDi:
2513 case AArch64::LDPQi:
2514 case AArch64::LDPSi:
2515 case AArch64::LDPWi:
2516 case AArch64::LDPXi:
2517 case AArch64::LDRBBpost:
2518 case AArch64::LDRBBpre:
2519 case AArch64::LDRBpost:
2520 case AArch64::LDRBpre:
2521 case AArch64::LDRDpost:
2522 case AArch64::LDRDpre:
2523 case AArch64::LDRHHpost:
2524 case AArch64::LDRHHpre:
2525 case AArch64::LDRHpost:
2526 case AArch64::LDRHpre:
2527 case AArch64::LDRQpost:
2528 case AArch64::LDRQpre:
2529 case AArch64::LDRSpost:
2530 case AArch64::LDRSpre:
2531 case AArch64::LDRWpost:
2532 case AArch64::LDRWpre:
2533 case AArch64::LDRXpost:
2534 case AArch64::LDRXpre:
2535 case AArch64::ST1B_D_IMM:
2536 case AArch64::ST1B_H_IMM:
2537 case AArch64::ST1B_IMM:
2538 case AArch64::ST1B_S_IMM:
2539 case AArch64::ST1D_IMM:
2540 case AArch64::ST1H_D_IMM:
2541 case AArch64::ST1H_IMM:
2542 case AArch64::ST1H_S_IMM:
2543 case AArch64::ST1W_D_IMM:
2544 case AArch64::ST1W_IMM:
2545 case AArch64::ST2B_IMM:
2546 case AArch64::ST2D_IMM:
2547 case AArch64::ST2H_IMM:
2548 case AArch64::ST2W_IMM:
2549 case AArch64::ST3B_IMM:
2550 case AArch64::ST3D_IMM:
2551 case AArch64::ST3H_IMM:
2552 case AArch64::ST3W_IMM:
2553 case AArch64::ST4B_IMM:
2554 case AArch64::ST4D_IMM:
2555 case AArch64::ST4H_IMM:
2556 case AArch64::ST4W_IMM:
2557 case AArch64::STGPi:
2558 case AArch64::STGPreIndex:
2559 case AArch64::STZGPreIndex:
2560 case AArch64::ST2GPreIndex:
2561 case AArch64::STZ2GPreIndex:
2562 case AArch64::STGPostIndex:
2563 case AArch64::STZGPostIndex:
2564 case AArch64::ST2GPostIndex:
2565 case AArch64::STZ2GPostIndex:
2566 case AArch64::STNPDi:
2567 case AArch64::STNPQi:
2568 case AArch64::STNPSi:
2569 case AArch64::STNPWi:
2570 case AArch64::STNPXi:
2571 case AArch64::STNT1B_ZRI:
2572 case AArch64::STNT1D_ZRI:
2573 case AArch64::STNT1H_ZRI:
2574 case AArch64::STNT1W_ZRI:
2575 case AArch64::STPDi:
2576 case AArch64::STPQi:
2577 case AArch64::STPSi:
2578 case AArch64::STPWi:
2579 case AArch64::STPXi:
2580 case AArch64::STRBBpost:
2581 case AArch64::STRBBpre:
2582 case AArch64::STRBpost:
2583 case AArch64::STRBpre:
2584 case AArch64::STRDpost:
2585 case AArch64::STRDpre:
2586 case AArch64::STRHHpost:
2587 case AArch64::STRHHpre:
2588 case AArch64::STRHpost:
2589 case AArch64::STRHpre:
2590 case AArch64::STRQpost:
2591 case AArch64::STRQpre:
2592 case AArch64::STRSpost:
2593 case AArch64::STRSpre:
2594 case AArch64::STRWpost:
2595 case AArch64::STRWpre:
2596 case AArch64::STRXpost:
2597 case AArch64::STRXpre:
2599 case AArch64::LDPDpost:
2600 case AArch64::LDPDpre:
2601 case AArch64::LDPQpost:
2602 case AArch64::LDPQpre:
2603 case AArch64::LDPSpost:
2604 case AArch64::LDPSpre:
2605 case AArch64::LDPWpost:
2606 case AArch64::LDPWpre:
2607 case AArch64::LDPXpost:
2608 case AArch64::LDPXpre:
2609 case AArch64::STGPpre:
2610 case AArch64::STGPpost:
2611 case AArch64::STPDpost:
2612 case AArch64::STPDpre:
2613 case AArch64::STPQpost:
2614 case AArch64::STPQpre:
2615 case AArch64::STPSpost:
2616 case AArch64::STPSpre:
2617 case AArch64::STPWpost:
2618 case AArch64::STPWpre:
2619 case AArch64::STPXpost:
2620 case AArch64::STPXpre:
2626 switch (
MI.getOpcode()) {
2630 case AArch64::STRSui:
2631 case AArch64::STRDui:
2632 case AArch64::STRQui:
2633 case AArch64::STRXui:
2634 case AArch64::STRWui:
2635 case AArch64::LDRSui:
2636 case AArch64::LDRDui:
2637 case AArch64::LDRQui:
2638 case AArch64::LDRXui:
2639 case AArch64::LDRWui:
2640 case AArch64::LDRSWui:
2642 case AArch64::STURSi:
2643 case AArch64::STRSpre:
2644 case AArch64::STURDi:
2645 case AArch64::STRDpre:
2646 case AArch64::STURQi:
2647 case AArch64::STRQpre:
2648 case AArch64::STURWi:
2649 case AArch64::STRWpre:
2650 case AArch64::STURXi:
2651 case AArch64::STRXpre:
2652 case AArch64::LDURSi:
2653 case AArch64::LDRSpre:
2654 case AArch64::LDURDi:
2655 case AArch64::LDRDpre:
2656 case AArch64::LDURQi:
2657 case AArch64::LDRQpre:
2658 case AArch64::LDURWi:
2659 case AArch64::LDRWpre:
2660 case AArch64::LDURXi:
2661 case AArch64::LDRXpre:
2662 case AArch64::LDURSWi:
2663 case AArch64::LDRSWpre:
2669 switch (
MI.getOpcode()) {
2672 "Unexpected instruction - was a new tail call opcode introduced?");
2674 case AArch64::TCRETURNdi:
2675 case AArch64::TCRETURNri:
2676 case AArch64::TCRETURNrix16x17:
2677 case AArch64::TCRETURNrix17:
2678 case AArch64::TCRETURNrinotx16:
2679 case AArch64::TCRETURNriALL:
2680 case AArch64::AUTH_TCRETURN:
2681 case AArch64::AUTH_TCRETURN_BTI:
2691 case AArch64::ADDWri:
2692 return AArch64::ADDSWri;
2693 case AArch64::ADDWrr:
2694 return AArch64::ADDSWrr;
2695 case AArch64::ADDWrs:
2696 return AArch64::ADDSWrs;
2697 case AArch64::ADDWrx:
2698 return AArch64::ADDSWrx;
2699 case AArch64::ANDWri:
2700 return AArch64::ANDSWri;
2701 case AArch64::ANDWrr:
2702 return AArch64::ANDSWrr;
2703 case AArch64::ANDWrs:
2704 return AArch64::ANDSWrs;
2705 case AArch64::BICWrr:
2706 return AArch64::BICSWrr;
2707 case AArch64::BICWrs:
2708 return AArch64::BICSWrs;
2709 case AArch64::SUBWri:
2710 return AArch64::SUBSWri;
2711 case AArch64::SUBWrr:
2712 return AArch64::SUBSWrr;
2713 case AArch64::SUBWrs:
2714 return AArch64::SUBSWrs;
2715 case AArch64::SUBWrx:
2716 return AArch64::SUBSWrx;
2718 case AArch64::ADDXri:
2719 return AArch64::ADDSXri;
2720 case AArch64::ADDXrr:
2721 return AArch64::ADDSXrr;
2722 case AArch64::ADDXrs:
2723 return AArch64::ADDSXrs;
2724 case AArch64::ADDXrx:
2725 return AArch64::ADDSXrx;
2726 case AArch64::ANDXri:
2727 return AArch64::ANDSXri;
2728 case AArch64::ANDXrr:
2729 return AArch64::ANDSXrr;
2730 case AArch64::ANDXrs:
2731 return AArch64::ANDSXrs;
2732 case AArch64::BICXrr:
2733 return AArch64::BICSXrr;
2734 case AArch64::BICXrs:
2735 return AArch64::BICSXrs;
2736 case AArch64::SUBXri:
2737 return AArch64::SUBSXri;
2738 case AArch64::SUBXrr:
2739 return AArch64::SUBSXrr;
2740 case AArch64::SUBXrs:
2741 return AArch64::SUBSXrs;
2742 case AArch64::SUBXrx:
2743 return AArch64::SUBSXrx;
2745 case AArch64::AND_PPzPP:
2746 return AArch64::ANDS_PPzPP;
2747 case AArch64::BIC_PPzPP:
2748 return AArch64::BICS_PPzPP;
2749 case AArch64::EOR_PPzPP:
2750 return AArch64::EORS_PPzPP;
2751 case AArch64::NAND_PPzPP:
2752 return AArch64::NANDS_PPzPP;
2753 case AArch64::NOR_PPzPP:
2754 return AArch64::NORS_PPzPP;
2755 case AArch64::ORN_PPzPP:
2756 return AArch64::ORNS_PPzPP;
2757 case AArch64::ORR_PPzPP:
2758 return AArch64::ORRS_PPzPP;
2759 case AArch64::BRKA_PPzP:
2760 return AArch64::BRKAS_PPzP;
2761 case AArch64::BRKPA_PPzPP:
2762 return AArch64::BRKPAS_PPzPP;
2763 case AArch64::BRKB_PPzP:
2764 return AArch64::BRKBS_PPzP;
2765 case AArch64::BRKPB_PPzPP:
2766 return AArch64::BRKPBS_PPzPP;
2767 case AArch64::BRKN_PPzP:
2768 return AArch64::BRKNS_PPzP;
2769 case AArch64::RDFFR_PPz:
2770 return AArch64::RDFFRS_PPz;
2771 case AArch64::PTRUE_B:
2772 return AArch64::PTRUES_B;
2783 if (
MI.hasOrderedMemoryRef())
2788 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2789 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2790 "Expected a reg or frame index operand.");
2794 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2796 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2809 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2810 Register BaseReg =
MI.getOperand(1).getReg();
2812 if (
MI.modifiesRegister(BaseReg,
TRI))
2825 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2827 MI.getMF()->getFunction().needsUnwindTableEntry();
2833 if (Subtarget.isPaired128Slow()) {
2834 switch (
MI.getOpcode()) {
2837 case AArch64::LDURQi:
2838 case AArch64::STURQi:
2839 case AArch64::LDRQui:
2840 case AArch64::STRQui:
2867std::optional<ExtAddrMode>
2872 bool OffsetIsScalable;
2873 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2874 return std::nullopt;
2877 return std::nullopt;
2892 int64_t OffsetScale = 1;
2897 case AArch64::LDURQi:
2898 case AArch64::STURQi:
2902 case AArch64::LDURDi:
2903 case AArch64::STURDi:
2904 case AArch64::LDURXi:
2905 case AArch64::STURXi:
2909 case AArch64::LDURWi:
2910 case AArch64::LDURSWi:
2911 case AArch64::STURWi:
2915 case AArch64::LDURHi:
2916 case AArch64::STURHi:
2917 case AArch64::LDURHHi:
2918 case AArch64::STURHHi:
2919 case AArch64::LDURSHXi:
2920 case AArch64::LDURSHWi:
2924 case AArch64::LDRBroX:
2925 case AArch64::LDRBBroX:
2926 case AArch64::LDRSBXroX:
2927 case AArch64::LDRSBWroX:
2928 case AArch64::STRBroX:
2929 case AArch64::STRBBroX:
2930 case AArch64::LDURBi:
2931 case AArch64::LDURBBi:
2932 case AArch64::LDURSBXi:
2933 case AArch64::LDURSBWi:
2934 case AArch64::STURBi:
2935 case AArch64::STURBBi:
2936 case AArch64::LDRBui:
2937 case AArch64::LDRBBui:
2938 case AArch64::LDRSBXui:
2939 case AArch64::LDRSBWui:
2940 case AArch64::STRBui:
2941 case AArch64::STRBBui:
2945 case AArch64::LDRQroX:
2946 case AArch64::STRQroX:
2947 case AArch64::LDRQui:
2948 case AArch64::STRQui:
2953 case AArch64::LDRDroX:
2954 case AArch64::STRDroX:
2955 case AArch64::LDRXroX:
2956 case AArch64::STRXroX:
2957 case AArch64::LDRDui:
2958 case AArch64::STRDui:
2959 case AArch64::LDRXui:
2960 case AArch64::STRXui:
2965 case AArch64::LDRWroX:
2966 case AArch64::LDRSWroX:
2967 case AArch64::STRWroX:
2968 case AArch64::LDRWui:
2969 case AArch64::LDRSWui:
2970 case AArch64::STRWui:
2975 case AArch64::LDRHroX:
2976 case AArch64::STRHroX:
2977 case AArch64::LDRHHroX:
2978 case AArch64::STRHHroX:
2979 case AArch64::LDRSHXroX:
2980 case AArch64::LDRSHWroX:
2981 case AArch64::LDRHui:
2982 case AArch64::STRHui:
2983 case AArch64::LDRHHui:
2984 case AArch64::STRHHui:
2985 case AArch64::LDRSHXui:
2986 case AArch64::LDRSHWui:
2994 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
3018 case AArch64::SBFMXri:
3031 AM.
Scale = OffsetScale;
3036 case TargetOpcode::SUBREG_TO_REG: {
3049 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3053 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3055 DefMI.getOperand(3).getImm() != 0)
3062 AM.
Scale = OffsetScale;
3073 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3074 int64_t NewOffset) ->
bool {
3075 int64_t MinOffset, MaxOffset;
3092 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3093 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3095 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3097 int64_t NewOffset = OldOffset + Disp;
3102 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3112 auto canFoldAddRegIntoAddrMode =
3129 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3130 Subtarget.isSTRQroSlow();
3139 case AArch64::ADDXri:
3145 return canFoldAddSubImmIntoAddrMode(Disp);
3147 case AArch64::SUBXri:
3153 return canFoldAddSubImmIntoAddrMode(-Disp);
3155 case AArch64::ADDXrs: {
3168 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3170 if (avoidSlowSTRQ(MemI))
3173 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3176 case AArch64::ADDXrr:
3184 if (!OptSize && avoidSlowSTRQ(MemI))
3186 return canFoldAddRegIntoAddrMode(1);
3188 case AArch64::ADDXrx:
3196 if (!OptSize && avoidSlowSTRQ(MemI))
3205 return canFoldAddRegIntoAddrMode(
3220 case AArch64::LDURQi:
3221 case AArch64::LDRQui:
3222 return AArch64::LDRQroX;
3223 case AArch64::STURQi:
3224 case AArch64::STRQui:
3225 return AArch64::STRQroX;
3226 case AArch64::LDURDi:
3227 case AArch64::LDRDui:
3228 return AArch64::LDRDroX;
3229 case AArch64::STURDi:
3230 case AArch64::STRDui:
3231 return AArch64::STRDroX;
3232 case AArch64::LDURXi:
3233 case AArch64::LDRXui:
3234 return AArch64::LDRXroX;
3235 case AArch64::STURXi:
3236 case AArch64::STRXui:
3237 return AArch64::STRXroX;
3238 case AArch64::LDURWi:
3239 case AArch64::LDRWui:
3240 return AArch64::LDRWroX;
3241 case AArch64::LDURSWi:
3242 case AArch64::LDRSWui:
3243 return AArch64::LDRSWroX;
3244 case AArch64::STURWi:
3245 case AArch64::STRWui:
3246 return AArch64::STRWroX;
3247 case AArch64::LDURHi:
3248 case AArch64::LDRHui:
3249 return AArch64::LDRHroX;
3250 case AArch64::STURHi:
3251 case AArch64::STRHui:
3252 return AArch64::STRHroX;
3253 case AArch64::LDURHHi:
3254 case AArch64::LDRHHui:
3255 return AArch64::LDRHHroX;
3256 case AArch64::STURHHi:
3257 case AArch64::STRHHui:
3258 return AArch64::STRHHroX;
3259 case AArch64::LDURSHXi:
3260 case AArch64::LDRSHXui:
3261 return AArch64::LDRSHXroX;
3262 case AArch64::LDURSHWi:
3263 case AArch64::LDRSHWui:
3264 return AArch64::LDRSHWroX;
3265 case AArch64::LDURBi:
3266 case AArch64::LDRBui:
3267 return AArch64::LDRBroX;
3268 case AArch64::LDURBBi:
3269 case AArch64::LDRBBui:
3270 return AArch64::LDRBBroX;
3271 case AArch64::LDURSBXi:
3272 case AArch64::LDRSBXui:
3273 return AArch64::LDRSBXroX;
3274 case AArch64::LDURSBWi:
3275 case AArch64::LDRSBWui:
3276 return AArch64::LDRSBWroX;
3277 case AArch64::STURBi:
3278 case AArch64::STRBui:
3279 return AArch64::STRBroX;
3280 case AArch64::STURBBi:
3281 case AArch64::STRBBui:
3282 return AArch64::STRBBroX;
3294 case AArch64::LDURQi:
3296 return AArch64::LDRQui;
3297 case AArch64::STURQi:
3299 return AArch64::STRQui;
3300 case AArch64::LDURDi:
3302 return AArch64::LDRDui;
3303 case AArch64::STURDi:
3305 return AArch64::STRDui;
3306 case AArch64::LDURXi:
3308 return AArch64::LDRXui;
3309 case AArch64::STURXi:
3311 return AArch64::STRXui;
3312 case AArch64::LDURWi:
3314 return AArch64::LDRWui;
3315 case AArch64::LDURSWi:
3317 return AArch64::LDRSWui;
3318 case AArch64::STURWi:
3320 return AArch64::STRWui;
3321 case AArch64::LDURHi:
3323 return AArch64::LDRHui;
3324 case AArch64::STURHi:
3326 return AArch64::STRHui;
3327 case AArch64::LDURHHi:
3329 return AArch64::LDRHHui;
3330 case AArch64::STURHHi:
3332 return AArch64::STRHHui;
3333 case AArch64::LDURSHXi:
3335 return AArch64::LDRSHXui;
3336 case AArch64::LDURSHWi:
3338 return AArch64::LDRSHWui;
3339 case AArch64::LDURBi:
3341 return AArch64::LDRBui;
3342 case AArch64::LDURBBi:
3344 return AArch64::LDRBBui;
3345 case AArch64::LDURSBXi:
3347 return AArch64::LDRSBXui;
3348 case AArch64::LDURSBWi:
3350 return AArch64::LDRSBWui;
3351 case AArch64::STURBi:
3353 return AArch64::STRBui;
3354 case AArch64::STURBBi:
3356 return AArch64::STRBBui;
3357 case AArch64::LDRQui:
3358 case AArch64::STRQui:
3361 case AArch64::LDRDui:
3362 case AArch64::STRDui:
3363 case AArch64::LDRXui:
3364 case AArch64::STRXui:
3367 case AArch64::LDRWui:
3368 case AArch64::LDRSWui:
3369 case AArch64::STRWui:
3372 case AArch64::LDRHui:
3373 case AArch64::STRHui:
3374 case AArch64::LDRHHui:
3375 case AArch64::STRHHui:
3376 case AArch64::LDRSHXui:
3377 case AArch64::LDRSHWui:
3380 case AArch64::LDRBui:
3381 case AArch64::LDRBBui:
3382 case AArch64::LDRSBXui:
3383 case AArch64::LDRSBWui:
3384 case AArch64::STRBui:
3385 case AArch64::STRBBui:
3399 case AArch64::LDURQi:
3400 case AArch64::STURQi:
3401 case AArch64::LDURDi:
3402 case AArch64::STURDi:
3403 case AArch64::LDURXi:
3404 case AArch64::STURXi:
3405 case AArch64::LDURWi:
3406 case AArch64::LDURSWi:
3407 case AArch64::STURWi:
3408 case AArch64::LDURHi:
3409 case AArch64::STURHi:
3410 case AArch64::LDURHHi:
3411 case AArch64::STURHHi:
3412 case AArch64::LDURSHXi:
3413 case AArch64::LDURSHWi:
3414 case AArch64::LDURBi:
3415 case AArch64::STURBi:
3416 case AArch64::LDURBBi:
3417 case AArch64::STURBBi:
3418 case AArch64::LDURSBWi:
3419 case AArch64::LDURSBXi:
3421 case AArch64::LDRQui:
3422 return AArch64::LDURQi;
3423 case AArch64::STRQui:
3424 return AArch64::STURQi;
3425 case AArch64::LDRDui:
3426 return AArch64::LDURDi;
3427 case AArch64::STRDui:
3428 return AArch64::STURDi;
3429 case AArch64::LDRXui:
3430 return AArch64::LDURXi;
3431 case AArch64::STRXui:
3432 return AArch64::STURXi;
3433 case AArch64::LDRWui:
3434 return AArch64::LDURWi;
3435 case AArch64::LDRSWui:
3436 return AArch64::LDURSWi;
3437 case AArch64::STRWui:
3438 return AArch64::STURWi;
3439 case AArch64::LDRHui:
3440 return AArch64::LDURHi;
3441 case AArch64::STRHui:
3442 return AArch64::STURHi;
3443 case AArch64::LDRHHui:
3444 return AArch64::LDURHHi;
3445 case AArch64::STRHHui:
3446 return AArch64::STURHHi;
3447 case AArch64::LDRSHXui:
3448 return AArch64::LDURSHXi;
3449 case AArch64::LDRSHWui:
3450 return AArch64::LDURSHWi;
3451 case AArch64::LDRBBui:
3452 return AArch64::LDURBBi;
3453 case AArch64::LDRBui:
3454 return AArch64::LDURBi;
3455 case AArch64::STRBBui:
3456 return AArch64::STURBBi;
3457 case AArch64::STRBui:
3458 return AArch64::STURBi;
3459 case AArch64::LDRSBWui:
3460 return AArch64::LDURSBWi;
3461 case AArch64::LDRSBXui:
3462 return AArch64::LDURSBXi;
3475 case AArch64::LDRQroX:
3476 case AArch64::LDURQi:
3477 case AArch64::LDRQui:
3478 return AArch64::LDRQroW;
3479 case AArch64::STRQroX:
3480 case AArch64::STURQi:
3481 case AArch64::STRQui:
3482 return AArch64::STRQroW;
3483 case AArch64::LDRDroX:
3484 case AArch64::LDURDi:
3485 case AArch64::LDRDui:
3486 return AArch64::LDRDroW;
3487 case AArch64::STRDroX:
3488 case AArch64::STURDi:
3489 case AArch64::STRDui:
3490 return AArch64::STRDroW;
3491 case AArch64::LDRXroX:
3492 case AArch64::LDURXi:
3493 case AArch64::LDRXui:
3494 return AArch64::LDRXroW;
3495 case AArch64::STRXroX:
3496 case AArch64::STURXi:
3497 case AArch64::STRXui:
3498 return AArch64::STRXroW;
3499 case AArch64::LDRWroX:
3500 case AArch64::LDURWi:
3501 case AArch64::LDRWui:
3502 return AArch64::LDRWroW;
3503 case AArch64::LDRSWroX:
3504 case AArch64::LDURSWi:
3505 case AArch64::LDRSWui:
3506 return AArch64::LDRSWroW;
3507 case AArch64::STRWroX:
3508 case AArch64::STURWi:
3509 case AArch64::STRWui:
3510 return AArch64::STRWroW;
3511 case AArch64::LDRHroX:
3512 case AArch64::LDURHi:
3513 case AArch64::LDRHui:
3514 return AArch64::LDRHroW;
3515 case AArch64::STRHroX:
3516 case AArch64::STURHi:
3517 case AArch64::STRHui:
3518 return AArch64::STRHroW;
3519 case AArch64::LDRHHroX:
3520 case AArch64::LDURHHi:
3521 case AArch64::LDRHHui:
3522 return AArch64::LDRHHroW;
3523 case AArch64::STRHHroX:
3524 case AArch64::STURHHi:
3525 case AArch64::STRHHui:
3526 return AArch64::STRHHroW;
3527 case AArch64::LDRSHXroX:
3528 case AArch64::LDURSHXi:
3529 case AArch64::LDRSHXui:
3530 return AArch64::LDRSHXroW;
3531 case AArch64::LDRSHWroX:
3532 case AArch64::LDURSHWi:
3533 case AArch64::LDRSHWui:
3534 return AArch64::LDRSHWroW;
3535 case AArch64::LDRBroX:
3536 case AArch64::LDURBi:
3537 case AArch64::LDRBui:
3538 return AArch64::LDRBroW;
3539 case AArch64::LDRBBroX:
3540 case AArch64::LDURBBi:
3541 case AArch64::LDRBBui:
3542 return AArch64::LDRBBroW;
3543 case AArch64::LDRSBXroX:
3544 case AArch64::LDURSBXi:
3545 case AArch64::LDRSBXui:
3546 return AArch64::LDRSBXroW;
3547 case AArch64::LDRSBWroX:
3548 case AArch64::LDURSBWi:
3549 case AArch64::LDRSBWui:
3550 return AArch64::LDRSBWroW;
3551 case AArch64::STRBroX:
3552 case AArch64::STURBi:
3553 case AArch64::STRBui:
3554 return AArch64::STRBroW;
3555 case AArch64::STRBBroX:
3556 case AArch64::STURBBi:
3557 case AArch64::STRBBui:
3558 return AArch64::STRBBroW;
3573 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3583 return B.getInstr();
3587 "Addressing mode not supported for folding");
3604 return B.getInstr();
3611 "Address offset can be a register or an immediate, but not both");
3613 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3618 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3632 return B.getInstr();
3636 "Function must not be called with an addressing mode it can't handle");
3645 case AArch64::LD1Fourv16b_POST:
3646 case AArch64::LD1Fourv1d_POST:
3647 case AArch64::LD1Fourv2d_POST:
3648 case AArch64::LD1Fourv2s_POST:
3649 case AArch64::LD1Fourv4h_POST:
3650 case AArch64::LD1Fourv4s_POST:
3651 case AArch64::LD1Fourv8b_POST:
3652 case AArch64::LD1Fourv8h_POST:
3653 case AArch64::LD1Onev16b_POST:
3654 case AArch64::LD1Onev1d_POST:
3655 case AArch64::LD1Onev2d_POST:
3656 case AArch64::LD1Onev2s_POST:
3657 case AArch64::LD1Onev4h_POST:
3658 case AArch64::LD1Onev4s_POST:
3659 case AArch64::LD1Onev8b_POST:
3660 case AArch64::LD1Onev8h_POST:
3661 case AArch64::LD1Rv16b_POST:
3662 case AArch64::LD1Rv1d_POST:
3663 case AArch64::LD1Rv2d_POST:
3664 case AArch64::LD1Rv2s_POST:
3665 case AArch64::LD1Rv4h_POST:
3666 case AArch64::LD1Rv4s_POST:
3667 case AArch64::LD1Rv8b_POST:
3668 case AArch64::LD1Rv8h_POST:
3669 case AArch64::LD1Threev16b_POST:
3670 case AArch64::LD1Threev1d_POST:
3671 case AArch64::LD1Threev2d_POST:
3672 case AArch64::LD1Threev2s_POST:
3673 case AArch64::LD1Threev4h_POST:
3674 case AArch64::LD1Threev4s_POST:
3675 case AArch64::LD1Threev8b_POST:
3676 case AArch64::LD1Threev8h_POST:
3677 case AArch64::LD1Twov16b_POST:
3678 case AArch64::LD1Twov1d_POST:
3679 case AArch64::LD1Twov2d_POST:
3680 case AArch64::LD1Twov2s_POST:
3681 case AArch64::LD1Twov4h_POST:
3682 case AArch64::LD1Twov4s_POST:
3683 case AArch64::LD1Twov8b_POST:
3684 case AArch64::LD1Twov8h_POST:
3685 case AArch64::LD1i16_POST:
3686 case AArch64::LD1i32_POST:
3687 case AArch64::LD1i64_POST:
3688 case AArch64::LD1i8_POST:
3689 case AArch64::LD2Rv16b_POST:
3690 case AArch64::LD2Rv1d_POST:
3691 case AArch64::LD2Rv2d_POST:
3692 case AArch64::LD2Rv2s_POST:
3693 case AArch64::LD2Rv4h_POST:
3694 case AArch64::LD2Rv4s_POST:
3695 case AArch64::LD2Rv8b_POST:
3696 case AArch64::LD2Rv8h_POST:
3697 case AArch64::LD2Twov16b_POST:
3698 case AArch64::LD2Twov2d_POST:
3699 case AArch64::LD2Twov2s_POST:
3700 case AArch64::LD2Twov4h_POST:
3701 case AArch64::LD2Twov4s_POST:
3702 case AArch64::LD2Twov8b_POST:
3703 case AArch64::LD2Twov8h_POST:
3704 case AArch64::LD2i16_POST:
3705 case AArch64::LD2i32_POST:
3706 case AArch64::LD2i64_POST:
3707 case AArch64::LD2i8_POST:
3708 case AArch64::LD3Rv16b_POST:
3709 case AArch64::LD3Rv1d_POST:
3710 case AArch64::LD3Rv2d_POST:
3711 case AArch64::LD3Rv2s_POST:
3712 case AArch64::LD3Rv4h_POST:
3713 case AArch64::LD3Rv4s_POST:
3714 case AArch64::LD3Rv8b_POST:
3715 case AArch64::LD3Rv8h_POST:
3716 case AArch64::LD3Threev16b_POST:
3717 case AArch64::LD3Threev2d_POST:
3718 case AArch64::LD3Threev2s_POST:
3719 case AArch64::LD3Threev4h_POST:
3720 case AArch64::LD3Threev4s_POST:
3721 case AArch64::LD3Threev8b_POST:
3722 case AArch64::LD3Threev8h_POST:
3723 case AArch64::LD3i16_POST:
3724 case AArch64::LD3i32_POST:
3725 case AArch64::LD3i64_POST:
3726 case AArch64::LD3i8_POST:
3727 case AArch64::LD4Fourv16b_POST:
3728 case AArch64::LD4Fourv2d_POST:
3729 case AArch64::LD4Fourv2s_POST:
3730 case AArch64::LD4Fourv4h_POST:
3731 case AArch64::LD4Fourv4s_POST:
3732 case AArch64::LD4Fourv8b_POST:
3733 case AArch64::LD4Fourv8h_POST:
3734 case AArch64::LD4Rv16b_POST:
3735 case AArch64::LD4Rv1d_POST:
3736 case AArch64::LD4Rv2d_POST:
3737 case AArch64::LD4Rv2s_POST:
3738 case AArch64::LD4Rv4h_POST:
3739 case AArch64::LD4Rv4s_POST:
3740 case AArch64::LD4Rv8b_POST:
3741 case AArch64::LD4Rv8h_POST:
3742 case AArch64::LD4i16_POST:
3743 case AArch64::LD4i32_POST:
3744 case AArch64::LD4i64_POST:
3745 case AArch64::LD4i8_POST:
3746 case AArch64::LDAPRWpost:
3747 case AArch64::LDAPRXpost:
3748 case AArch64::LDIAPPWpost:
3749 case AArch64::LDIAPPXpost:
3750 case AArch64::LDPDpost:
3751 case AArch64::LDPQpost:
3752 case AArch64::LDPSWpost:
3753 case AArch64::LDPSpost:
3754 case AArch64::LDPWpost:
3755 case AArch64::LDPXpost:
3756 case AArch64::LDRBBpost:
3757 case AArch64::LDRBpost:
3758 case AArch64::LDRDpost:
3759 case AArch64::LDRHHpost:
3760 case AArch64::LDRHpost:
3761 case AArch64::LDRQpost:
3762 case AArch64::LDRSBWpost:
3763 case AArch64::LDRSBXpost:
3764 case AArch64::LDRSHWpost:
3765 case AArch64::LDRSHXpost:
3766 case AArch64::LDRSWpost:
3767 case AArch64::LDRSpost:
3768 case AArch64::LDRWpost:
3769 case AArch64::LDRXpost:
3770 case AArch64::ST1Fourv16b_POST:
3771 case AArch64::ST1Fourv1d_POST:
3772 case AArch64::ST1Fourv2d_POST:
3773 case AArch64::ST1Fourv2s_POST:
3774 case AArch64::ST1Fourv4h_POST:
3775 case AArch64::ST1Fourv4s_POST:
3776 case AArch64::ST1Fourv8b_POST:
3777 case AArch64::ST1Fourv8h_POST:
3778 case AArch64::ST1Onev16b_POST:
3779 case AArch64::ST1Onev1d_POST:
3780 case AArch64::ST1Onev2d_POST:
3781 case AArch64::ST1Onev2s_POST:
3782 case AArch64::ST1Onev4h_POST:
3783 case AArch64::ST1Onev4s_POST:
3784 case AArch64::ST1Onev8b_POST:
3785 case AArch64::ST1Onev8h_POST:
3786 case AArch64::ST1Threev16b_POST:
3787 case AArch64::ST1Threev1d_POST:
3788 case AArch64::ST1Threev2d_POST:
3789 case AArch64::ST1Threev2s_POST:
3790 case AArch64::ST1Threev4h_POST:
3791 case AArch64::ST1Threev4s_POST:
3792 case AArch64::ST1Threev8b_POST:
3793 case AArch64::ST1Threev8h_POST:
3794 case AArch64::ST1Twov16b_POST:
3795 case AArch64::ST1Twov1d_POST:
3796 case AArch64::ST1Twov2d_POST:
3797 case AArch64::ST1Twov2s_POST:
3798 case AArch64::ST1Twov4h_POST:
3799 case AArch64::ST1Twov4s_POST:
3800 case AArch64::ST1Twov8b_POST:
3801 case AArch64::ST1Twov8h_POST:
3802 case AArch64::ST1i16_POST:
3803 case AArch64::ST1i32_POST:
3804 case AArch64::ST1i64_POST:
3805 case AArch64::ST1i8_POST:
3806 case AArch64::ST2GPostIndex:
3807 case AArch64::ST2Twov16b_POST:
3808 case AArch64::ST2Twov2d_POST:
3809 case AArch64::ST2Twov2s_POST:
3810 case AArch64::ST2Twov4h_POST:
3811 case AArch64::ST2Twov4s_POST:
3812 case AArch64::ST2Twov8b_POST:
3813 case AArch64::ST2Twov8h_POST:
3814 case AArch64::ST2i16_POST:
3815 case AArch64::ST2i32_POST:
3816 case AArch64::ST2i64_POST:
3817 case AArch64::ST2i8_POST:
3818 case AArch64::ST3Threev16b_POST:
3819 case AArch64::ST3Threev2d_POST:
3820 case AArch64::ST3Threev2s_POST:
3821 case AArch64::ST3Threev4h_POST:
3822 case AArch64::ST3Threev4s_POST:
3823 case AArch64::ST3Threev8b_POST:
3824 case AArch64::ST3Threev8h_POST:
3825 case AArch64::ST3i16_POST:
3826 case AArch64::ST3i32_POST:
3827 case AArch64::ST3i64_POST:
3828 case AArch64::ST3i8_POST:
3829 case AArch64::ST4Fourv16b_POST:
3830 case AArch64::ST4Fourv2d_POST:
3831 case AArch64::ST4Fourv2s_POST:
3832 case AArch64::ST4Fourv4h_POST:
3833 case AArch64::ST4Fourv4s_POST:
3834 case AArch64::ST4Fourv8b_POST:
3835 case AArch64::ST4Fourv8h_POST:
3836 case AArch64::ST4i16_POST:
3837 case AArch64::ST4i32_POST:
3838 case AArch64::ST4i64_POST:
3839 case AArch64::ST4i8_POST:
3840 case AArch64::STGPostIndex:
3841 case AArch64::STGPpost:
3842 case AArch64::STPDpost:
3843 case AArch64::STPQpost:
3844 case AArch64::STPSpost:
3845 case AArch64::STPWpost:
3846 case AArch64::STPXpost:
3847 case AArch64::STRBBpost:
3848 case AArch64::STRBpost:
3849 case AArch64::STRDpost:
3850 case AArch64::STRHHpost:
3851 case AArch64::STRHpost:
3852 case AArch64::STRQpost:
3853 case AArch64::STRSpost:
3854 case AArch64::STRWpost:
3855 case AArch64::STRXpost:
3856 case AArch64::STZ2GPostIndex:
3857 case AArch64::STZGPostIndex:
3864 bool &OffsetIsScalable,
TypeSize &Width,
3885 int64_t Dummy1, Dummy2;
3907 return BaseOp->
isReg() || BaseOp->
isFI();
3914 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3919 TypeSize &Width, int64_t &MinOffset,
3920 int64_t &MaxOffset) {
3926 MinOffset = MaxOffset = 0;
3929 case AArch64::LDRQui:
3930 case AArch64::STRQui:
3936 case AArch64::LDRXui:
3937 case AArch64::LDRDui:
3938 case AArch64::STRXui:
3939 case AArch64::STRDui:
3940 case AArch64::PRFMui:
3946 case AArch64::LDRWui:
3947 case AArch64::LDRSui:
3948 case AArch64::LDRSWui:
3949 case AArch64::STRWui:
3950 case AArch64::STRSui:
3956 case AArch64::LDRHui:
3957 case AArch64::LDRHHui:
3958 case AArch64::LDRSHWui:
3959 case AArch64::LDRSHXui:
3960 case AArch64::STRHui:
3961 case AArch64::STRHHui:
3967 case AArch64::LDRBui:
3968 case AArch64::LDRBBui:
3969 case AArch64::LDRSBWui:
3970 case AArch64::LDRSBXui:
3971 case AArch64::STRBui:
3972 case AArch64::STRBBui:
3979 case AArch64::STRQpre:
3980 case AArch64::LDRQpost:
3986 case AArch64::LDRDpost:
3987 case AArch64::LDRDpre:
3988 case AArch64::LDRXpost:
3989 case AArch64::LDRXpre:
3990 case AArch64::STRDpost:
3991 case AArch64::STRDpre:
3992 case AArch64::STRXpost:
3993 case AArch64::STRXpre:
3999 case AArch64::STRWpost:
4000 case AArch64::STRWpre:
4001 case AArch64::LDRWpost:
4002 case AArch64::LDRWpre:
4003 case AArch64::STRSpost:
4004 case AArch64::STRSpre:
4005 case AArch64::LDRSpost:
4006 case AArch64::LDRSpre:
4012 case AArch64::LDRHpost:
4013 case AArch64::LDRHpre:
4014 case AArch64::STRHpost:
4015 case AArch64::STRHpre:
4016 case AArch64::LDRHHpost:
4017 case AArch64::LDRHHpre:
4018 case AArch64::STRHHpost:
4019 case AArch64::STRHHpre:
4025 case AArch64::LDRBpost:
4026 case AArch64::LDRBpre:
4027 case AArch64::STRBpost:
4028 case AArch64::STRBpre:
4029 case AArch64::LDRBBpost:
4030 case AArch64::LDRBBpre:
4031 case AArch64::STRBBpost:
4032 case AArch64::STRBBpre:
4039 case AArch64::LDURQi:
4040 case AArch64::STURQi:
4046 case AArch64::LDURXi:
4047 case AArch64::LDURDi:
4048 case AArch64::LDAPURXi:
4049 case AArch64::STURXi:
4050 case AArch64::STURDi:
4051 case AArch64::STLURXi:
4052 case AArch64::PRFUMi:
4058 case AArch64::LDURWi:
4059 case AArch64::LDURSi:
4060 case AArch64::LDURSWi:
4061 case AArch64::LDAPURi:
4062 case AArch64::LDAPURSWi:
4063 case AArch64::STURWi:
4064 case AArch64::STURSi:
4065 case AArch64::STLURWi:
4071 case AArch64::LDURHi:
4072 case AArch64::LDURHHi:
4073 case AArch64::LDURSHXi:
4074 case AArch64::LDURSHWi:
4075 case AArch64::LDAPURHi:
4076 case AArch64::LDAPURSHWi:
4077 case AArch64::LDAPURSHXi:
4078 case AArch64::STURHi:
4079 case AArch64::STURHHi:
4080 case AArch64::STLURHi:
4086 case AArch64::LDURBi:
4087 case AArch64::LDURBBi:
4088 case AArch64::LDURSBXi:
4089 case AArch64::LDURSBWi:
4090 case AArch64::LDAPURBi:
4091 case AArch64::LDAPURSBWi:
4092 case AArch64::LDAPURSBXi:
4093 case AArch64::STURBi:
4094 case AArch64::STURBBi:
4095 case AArch64::STLURBi:
4102 case AArch64::LDPQi:
4103 case AArch64::LDNPQi:
4104 case AArch64::STPQi:
4105 case AArch64::STNPQi:
4106 case AArch64::LDPQpost:
4107 case AArch64::LDPQpre:
4108 case AArch64::STPQpost:
4109 case AArch64::STPQpre:
4115 case AArch64::LDPXi:
4116 case AArch64::LDPDi:
4117 case AArch64::LDNPXi:
4118 case AArch64::LDNPDi:
4119 case AArch64::STPXi:
4120 case AArch64::STPDi:
4121 case AArch64::STNPXi:
4122 case AArch64::STNPDi:
4123 case AArch64::LDPDpost:
4124 case AArch64::LDPDpre:
4125 case AArch64::LDPXpost:
4126 case AArch64::LDPXpre:
4127 case AArch64::STPDpost:
4128 case AArch64::STPDpre:
4129 case AArch64::STPXpost:
4130 case AArch64::STPXpre:
4136 case AArch64::LDPWi:
4137 case AArch64::LDPSi:
4138 case AArch64::LDNPWi:
4139 case AArch64::LDNPSi:
4140 case AArch64::STPWi:
4141 case AArch64::STPSi:
4142 case AArch64::STNPWi:
4143 case AArch64::STNPSi:
4144 case AArch64::LDPSpost:
4145 case AArch64::LDPSpre:
4146 case AArch64::LDPWpost:
4147 case AArch64::LDPWpre:
4148 case AArch64::STPSpost:
4149 case AArch64::STPSpre:
4150 case AArch64::STPWpost:
4151 case AArch64::STPWpre:
4157 case AArch64::StoreSwiftAsyncContext:
4170 case AArch64::TAGPstack:
4180 case AArch64::STGPreIndex:
4181 case AArch64::STGPostIndex:
4182 case AArch64::STZGi:
4183 case AArch64::STZGPreIndex:
4184 case AArch64::STZGPostIndex:
4191 case AArch64::STR_ZZZZXI:
4192 case AArch64::LDR_ZZZZXI:
4198 case AArch64::STR_ZZZXI:
4199 case AArch64::LDR_ZZZXI:
4205 case AArch64::STR_ZZXI:
4206 case AArch64::LDR_ZZXI:
4212 case AArch64::LDR_PXI:
4213 case AArch64::STR_PXI:
4219 case AArch64::LDR_PPXI:
4220 case AArch64::STR_PPXI:
4226 case AArch64::LDR_ZXI:
4227 case AArch64::STR_ZXI:
4233 case AArch64::LD1B_IMM:
4234 case AArch64::LD1H_IMM:
4235 case AArch64::LD1W_IMM:
4236 case AArch64::LD1D_IMM:
4237 case AArch64::LDNT1B_ZRI:
4238 case AArch64::LDNT1H_ZRI:
4239 case AArch64::LDNT1W_ZRI:
4240 case AArch64::LDNT1D_ZRI:
4241 case AArch64::ST1B_IMM:
4242 case AArch64::ST1H_IMM:
4243 case AArch64::ST1W_IMM:
4244 case AArch64::ST1D_IMM:
4245 case AArch64::STNT1B_ZRI:
4246 case AArch64::STNT1H_ZRI:
4247 case AArch64::STNT1W_ZRI:
4248 case AArch64::STNT1D_ZRI:
4249 case AArch64::LDNF1B_IMM:
4250 case AArch64::LDNF1H_IMM:
4251 case AArch64::LDNF1W_IMM:
4252 case AArch64::LDNF1D_IMM:
4260 case AArch64::LD2B_IMM:
4261 case AArch64::LD2H_IMM:
4262 case AArch64::LD2W_IMM:
4263 case AArch64::LD2D_IMM:
4264 case AArch64::ST2B_IMM:
4265 case AArch64::ST2H_IMM:
4266 case AArch64::ST2W_IMM:
4267 case AArch64::ST2D_IMM:
4273 case AArch64::LD3B_IMM:
4274 case AArch64::LD3H_IMM:
4275 case AArch64::LD3W_IMM:
4276 case AArch64::LD3D_IMM:
4277 case AArch64::ST3B_IMM:
4278 case AArch64::ST3H_IMM:
4279 case AArch64::ST3W_IMM:
4280 case AArch64::ST3D_IMM:
4286 case AArch64::LD4B_IMM:
4287 case AArch64::LD4H_IMM:
4288 case AArch64::LD4W_IMM:
4289 case AArch64::LD4D_IMM:
4290 case AArch64::ST4B_IMM:
4291 case AArch64::ST4H_IMM:
4292 case AArch64::ST4W_IMM:
4293 case AArch64::ST4D_IMM:
4299 case AArch64::LD1B_H_IMM:
4300 case AArch64::LD1SB_H_IMM:
4301 case AArch64::LD1H_S_IMM:
4302 case AArch64::LD1SH_S_IMM:
4303 case AArch64::LD1W_D_IMM:
4304 case AArch64::LD1SW_D_IMM:
4305 case AArch64::ST1B_H_IMM:
4306 case AArch64::ST1H_S_IMM:
4307 case AArch64::ST1W_D_IMM:
4308 case AArch64::LDNF1B_H_IMM:
4309 case AArch64::LDNF1SB_H_IMM:
4310 case AArch64::LDNF1H_S_IMM:
4311 case AArch64::LDNF1SH_S_IMM:
4312 case AArch64::LDNF1W_D_IMM:
4313 case AArch64::LDNF1SW_D_IMM:
4321 case AArch64::LD1B_S_IMM:
4322 case AArch64::LD1SB_S_IMM:
4323 case AArch64::LD1H_D_IMM:
4324 case AArch64::LD1SH_D_IMM:
4325 case AArch64::ST1B_S_IMM:
4326 case AArch64::ST1H_D_IMM:
4327 case AArch64::LDNF1B_S_IMM:
4328 case AArch64::LDNF1SB_S_IMM:
4329 case AArch64::LDNF1H_D_IMM:
4330 case AArch64::LDNF1SH_D_IMM:
4338 case AArch64::LD1B_D_IMM:
4339 case AArch64::LD1SB_D_IMM:
4340 case AArch64::ST1B_D_IMM:
4341 case AArch64::LDNF1B_D_IMM:
4342 case AArch64::LDNF1SB_D_IMM:
4350 case AArch64::ST2Gi:
4351 case AArch64::ST2GPreIndex:
4352 case AArch64::ST2GPostIndex:
4353 case AArch64::STZ2Gi:
4354 case AArch64::STZ2GPreIndex:
4355 case AArch64::STZ2GPostIndex:
4361 case AArch64::STGPi:
4362 case AArch64::STGPpost:
4363 case AArch64::STGPpre:
4369 case AArch64::LD1RB_IMM:
4370 case AArch64::LD1RB_H_IMM:
4371 case AArch64::LD1RB_S_IMM:
4372 case AArch64::LD1RB_D_IMM:
4373 case AArch64::LD1RSB_H_IMM:
4374 case AArch64::LD1RSB_S_IMM:
4375 case AArch64::LD1RSB_D_IMM:
4381 case AArch64::LD1RH_IMM:
4382 case AArch64::LD1RH_S_IMM:
4383 case AArch64::LD1RH_D_IMM:
4384 case AArch64::LD1RSH_S_IMM:
4385 case AArch64::LD1RSH_D_IMM:
4391 case AArch64::LD1RW_IMM:
4392 case AArch64::LD1RW_D_IMM:
4393 case AArch64::LD1RSW_IMM:
4399 case AArch64::LD1RD_IMM:
4415 case AArch64::LDRBBui:
4416 case AArch64::LDURBBi:
4417 case AArch64::LDRSBWui:
4418 case AArch64::LDURSBWi:
4419 case AArch64::STRBBui:
4420 case AArch64::STURBBi:
4422 case AArch64::LDRHHui:
4423 case AArch64::LDURHHi:
4424 case AArch64::LDRSHWui:
4425 case AArch64::LDURSHWi:
4426 case AArch64::STRHHui:
4427 case AArch64::STURHHi:
4429 case AArch64::LDRSui:
4430 case AArch64::LDURSi:
4431 case AArch64::LDRSpre:
4432 case AArch64::LDRSWui:
4433 case AArch64::LDURSWi:
4434 case AArch64::LDRSWpre:
4435 case AArch64::LDRWpre:
4436 case AArch64::LDRWui:
4437 case AArch64::LDURWi:
4438 case AArch64::STRSui:
4439 case AArch64::STURSi:
4440 case AArch64::STRSpre:
4441 case AArch64::STRWui:
4442 case AArch64::STURWi:
4443 case AArch64::STRWpre:
4444 case AArch64::LDPSi:
4445 case AArch64::LDPSWi:
4446 case AArch64::LDPWi:
4447 case AArch64::STPSi:
4448 case AArch64::STPWi:
4450 case AArch64::LDRDui:
4451 case AArch64::LDURDi:
4452 case AArch64::LDRDpre:
4453 case AArch64::LDRXui:
4454 case AArch64::LDURXi:
4455 case AArch64::LDRXpre:
4456 case AArch64::STRDui:
4457 case AArch64::STURDi:
4458 case AArch64::STRDpre:
4459 case AArch64::STRXui:
4460 case AArch64::STURXi:
4461 case AArch64::STRXpre:
4462 case AArch64::LDPDi:
4463 case AArch64::LDPXi:
4464 case AArch64::STPDi:
4465 case AArch64::STPXi:
4467 case AArch64::LDRQui:
4468 case AArch64::LDURQi:
4469 case AArch64::STRQui:
4470 case AArch64::STURQi:
4471 case AArch64::STRQpre:
4472 case AArch64::LDPQi:
4473 case AArch64::LDRQpre:
4474 case AArch64::STPQi:
4476 case AArch64::STZGi:
4477 case AArch64::ST2Gi:
4478 case AArch64::STZ2Gi:
4479 case AArch64::STGPi:
4485 switch (
MI.getOpcode()) {
4488 case AArch64::LDRWpre:
4489 case AArch64::LDRXpre:
4490 case AArch64::LDRSWpre:
4491 case AArch64::LDRSpre:
4492 case AArch64::LDRDpre:
4493 case AArch64::LDRQpre:
4499 switch (
MI.getOpcode()) {
4502 case AArch64::STRWpre:
4503 case AArch64::STRXpre:
4504 case AArch64::STRSpre:
4505 case AArch64::STRDpre:
4506 case AArch64::STRQpre:
4516 switch (
MI.getOpcode()) {
4519 case AArch64::LDPSi:
4520 case AArch64::LDPSWi:
4521 case AArch64::LDPDi:
4522 case AArch64::LDPQi:
4523 case AArch64::LDPWi:
4524 case AArch64::LDPXi:
4525 case AArch64::STPSi:
4526 case AArch64::STPDi:
4527 case AArch64::STPQi:
4528 case AArch64::STPWi:
4529 case AArch64::STPXi:
4530 case AArch64::STGPi:
4536 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4540 return MI.getOperand(
Idx);
4545 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4549 return MI.getOperand(
Idx);
4554 switch (
MI.getOpcode()) {
4557 case AArch64::LDRBroX:
4558 case AArch64::LDRBBroX:
4559 case AArch64::LDRSBXroX:
4560 case AArch64::LDRSBWroX:
4561 case AArch64::LDRHroX:
4562 case AArch64::LDRHHroX:
4563 case AArch64::LDRSHXroX:
4564 case AArch64::LDRSHWroX:
4565 case AArch64::LDRWroX:
4566 case AArch64::LDRSroX:
4567 case AArch64::LDRSWroX:
4568 case AArch64::LDRDroX:
4569 case AArch64::LDRXroX:
4570 case AArch64::LDRQroX:
4571 return MI.getOperand(4);
4577 if (
MI.getParent() ==
nullptr)
4587 auto Reg =
Op.getReg();
4588 if (Reg.isPhysical())
4589 return AArch64::FPR16RegClass.
contains(Reg);
4591 return TRC == &AArch64::FPR16RegClass ||
4592 TRC == &AArch64::FPR16_loRegClass;
4601 auto Reg =
Op.getReg();
4602 if (Reg.isPhysical())
4603 return AArch64::FPR128RegClass.
contains(Reg);
4605 return TRC == &AArch64::FPR128RegClass ||
4606 TRC == &AArch64::FPR128_loRegClass;
4612 switch (
MI.getOpcode()) {
4615 case AArch64::PACIASP:
4616 case AArch64::PACIBSP:
4619 case AArch64::PAUTH_PROLOGUE:
4622 case AArch64::HINT: {
4623 unsigned Imm =
MI.getOperand(0).getImm();
4625 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4628 if (Imm == 25 || Imm == 27)
4640 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4641 return AArch64::FPR128RegClass.contains(Reg) ||
4642 AArch64::FPR64RegClass.contains(Reg) ||
4643 AArch64::FPR32RegClass.contains(Reg) ||
4644 AArch64::FPR16RegClass.contains(Reg) ||
4645 AArch64::FPR8RegClass.contains(Reg);
4652 auto Reg =
Op.getReg();
4653 if (Reg.isPhysical())
4657 return TRC == &AArch64::FPR128RegClass ||
4658 TRC == &AArch64::FPR128_loRegClass ||
4659 TRC == &AArch64::FPR64RegClass ||
4660 TRC == &AArch64::FPR64_loRegClass ||
4661 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4662 TRC == &AArch64::FPR8RegClass;
4684 if (FirstOpc == SecondOpc)
4690 case AArch64::STRSui:
4691 case AArch64::STURSi:
4692 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4693 case AArch64::STRDui:
4694 case AArch64::STURDi:
4695 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4696 case AArch64::STRQui:
4697 case AArch64::STURQi:
4698 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4699 case AArch64::STRWui:
4700 case AArch64::STURWi:
4701 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4702 case AArch64::STRXui:
4703 case AArch64::STURXi:
4704 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4705 case AArch64::LDRSui:
4706 case AArch64::LDURSi:
4707 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4708 case AArch64::LDRDui:
4709 case AArch64::LDURDi:
4710 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4711 case AArch64::LDRQui:
4712 case AArch64::LDURQi:
4713 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4714 case AArch64::LDRWui:
4715 case AArch64::LDURWi:
4716 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4717 case AArch64::LDRSWui:
4718 case AArch64::LDURSWi:
4719 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4720 case AArch64::LDRXui:
4721 case AArch64::LDURXi:
4722 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4729 int64_t Offset1,
unsigned Opcode1,
int FI2,
4730 int64_t Offset2,
unsigned Opcode2) {
4736 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4739 if (ObjectOffset1 % Scale1 != 0)
4741 ObjectOffset1 /= Scale1;
4743 if (ObjectOffset2 % Scale2 != 0)
4745 ObjectOffset2 /= Scale2;
4746 ObjectOffset1 += Offset1;
4747 ObjectOffset2 += Offset2;
4748 return ObjectOffset1 + 1 == ObjectOffset2;
4760 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4761 unsigned NumBytes)
const {
4771 "Only base registers and frame indices are supported.");
4778 if (ClusterSize > 2)
4785 unsigned FirstOpc = FirstLdSt.
getOpcode();
4786 unsigned SecondOpc = SecondLdSt.
getOpcode();
4806 if (Offset1 > 63 || Offset1 < -64)
4811 if (BaseOp1.
isFI()) {
4813 "Caller should have ordered offsets.");
4818 BaseOp2.
getIndex(), Offset2, SecondOpc);
4821 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4823 return Offset1 + 1 == Offset2;
4831 return MIB.
addReg(Reg, State);
4834 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4835 return MIB.
addReg(Reg, State, SubIdx);
4842 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4851 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4853 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4854 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4855 unsigned NumRegs = Indices.
size();
4857 int SubReg = 0,
End = NumRegs, Incr = 1;
4876 unsigned Opcode,
unsigned ZeroReg,
4879 unsigned NumRegs = Indices.
size();
4882 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4883 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4884 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4885 "GPR reg sequences should not be able to overlap");
4902 bool RenamableSrc)
const {
4903 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4904 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4907 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4909 if (Subtarget.hasZeroCycleRegMove()) {
4912 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4914 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4930 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4935 if (Subtarget.hasZeroCycleRegMove()) {
4938 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4940 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4960 if (AArch64::PPRRegClass.
contains(DestReg) &&
4961 AArch64::PPRRegClass.
contains(SrcReg)) {
4963 "Unexpected SVE register.");
4973 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4974 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4975 if (DestIsPNR || SrcIsPNR) {
4977 return (R - AArch64::PN0) + AArch64::P0;
4979 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4980 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4982 if (PPRSrcReg != PPRDestReg) {
4994 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4995 AArch64::ZPRRegClass.
contains(SrcReg)) {
4997 "Unexpected SVE register.");
5005 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
5006 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
5007 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
5008 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
5010 "Unexpected SVE register.");
5011 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5018 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
5019 AArch64::ZPR3RegClass.
contains(SrcReg)) {
5021 "Unexpected SVE register.");
5022 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5030 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
5031 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
5032 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
5033 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5035 "Unexpected SVE register.");
5036 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5037 AArch64::zsub2, AArch64::zsub3};
5043 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5044 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
5045 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5051 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
5065 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5066 AArch64::DDDDRegClass.
contains(SrcReg)) {
5067 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5068 AArch64::dsub2, AArch64::dsub3};
5075 if (AArch64::DDDRegClass.
contains(DestReg) &&
5076 AArch64::DDDRegClass.
contains(SrcReg)) {
5077 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5085 if (AArch64::DDRegClass.
contains(DestReg) &&
5086 AArch64::DDRegClass.
contains(SrcReg)) {
5087 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5094 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5095 AArch64::QQQQRegClass.
contains(SrcReg)) {
5096 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5097 AArch64::qsub2, AArch64::qsub3};
5104 if (AArch64::QQQRegClass.
contains(DestReg) &&
5105 AArch64::QQQRegClass.
contains(SrcReg)) {
5106 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5114 if (AArch64::QQRegClass.
contains(DestReg) &&
5115 AArch64::QQRegClass.
contains(SrcReg)) {
5116 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5122 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5123 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5124 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5126 AArch64::XZR, Indices);
5130 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5131 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5132 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5134 AArch64::WZR, Indices);
5138 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5139 AArch64::FPR128RegClass.
contains(SrcReg)) {
5144 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5145 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5165 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5166 AArch64::FPR64RegClass.
contains(SrcReg)) {
5172 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5173 AArch64::FPR32RegClass.
contains(SrcReg)) {
5179 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5180 AArch64::FPR16RegClass.
contains(SrcReg)) {
5182 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
5184 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
5190 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5191 AArch64::FPR8RegClass.
contains(SrcReg)) {
5193 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
5195 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
5202 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5203 AArch64::GPR64RegClass.
contains(SrcReg)) {
5208 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5209 AArch64::FPR64RegClass.
contains(SrcReg)) {
5215 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5216 AArch64::GPR32RegClass.
contains(SrcReg)) {
5221 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5222 AArch64::FPR32RegClass.
contains(SrcReg)) {
5228 if (DestReg == AArch64::NZCV) {
5229 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5231 .
addImm(AArch64SysReg::NZCV)
5237 if (SrcReg == AArch64::NZCV) {
5238 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
5240 .
addImm(AArch64SysReg::NZCV)
5247 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
5248 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
5258 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
5263 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
5265 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
5278 Register SrcReg,
bool isKill,
int FI,
5293 switch (
TRI->getSpillSize(*RC)) {
5295 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5296 Opc = AArch64::STRBui;
5299 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5300 Opc = AArch64::STRHui;
5301 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
5302 AArch64::PPRRegClass.hasSubClassEq(RC)) {
5304 "Unexpected register store without SVE store instructions");
5305 Opc = AArch64::STR_PXI;
5311 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5312 Opc = AArch64::STRWui;
5316 assert(SrcReg != AArch64::WSP);
5317 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5318 Opc = AArch64::STRSui;
5319 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5320 Opc = AArch64::STR_PPXI;
5325 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5326 Opc = AArch64::STRXui;
5330 assert(SrcReg != AArch64::SP);
5331 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5332 Opc = AArch64::STRDui;
5333 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5335 get(AArch64::STPWi), SrcReg, isKill,
5336 AArch64::sube32, AArch64::subo32, FI, MMO);
5341 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5342 Opc = AArch64::STRQui;
5343 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5344 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5345 Opc = AArch64::ST1Twov1d;
5347 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5349 get(AArch64::STPXi), SrcReg, isKill,
5350 AArch64::sube64, AArch64::subo64, FI, MMO);
5352 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5354 "Unexpected register store without SVE store instructions");
5355 Opc = AArch64::STR_ZXI;
5360 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5361 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5362 Opc = AArch64::ST1Threev1d;
5367 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5368 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5369 Opc = AArch64::ST1Fourv1d;
5371 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5372 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5373 Opc = AArch64::ST1Twov2d;
5375 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5376 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5378 "Unexpected register store without SVE store instructions");
5379 Opc = AArch64::STR_ZZXI;
5384 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5385 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5386 Opc = AArch64::ST1Threev2d;
5388 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5390 "Unexpected register store without SVE store instructions");
5391 Opc = AArch64::STR_ZZZXI;
5396 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5397 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5398 Opc = AArch64::ST1Fourv2d;
5400 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5401 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5403 "Unexpected register store without SVE store instructions");
5404 Opc = AArch64::STR_ZZZZXI;
5409 assert(Opc &&
"Unknown register class");
5420 MI.addMemOperand(MMO);
5427 Register DestReg,
unsigned SubIdx0,
5428 unsigned SubIdx1,
int FI,
5432 bool IsUndef =
true;
5434 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
5436 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5465 switch (
TRI->getSpillSize(*RC)) {
5467 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5468 Opc = AArch64::LDRBui;
5471 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5472 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5473 Opc = AArch64::LDRHui;
5474 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5476 "Unexpected register load without SVE load instructions");
5479 Opc = AArch64::LDR_PXI;
5485 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5486 Opc = AArch64::LDRWui;
5490 assert(DestReg != AArch64::WSP);
5491 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5492 Opc = AArch64::LDRSui;
5493 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5494 Opc = AArch64::LDR_PPXI;
5499 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5500 Opc = AArch64::LDRXui;
5504 assert(DestReg != AArch64::SP);
5505 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5506 Opc = AArch64::LDRDui;
5507 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5509 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5510 AArch64::subo32, FI, MMO);
5515 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5516 Opc = AArch64::LDRQui;
5517 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5518 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5519 Opc = AArch64::LD1Twov1d;
5521 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5523 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5524 AArch64::subo64, FI, MMO);
5526 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5528 "Unexpected register load without SVE load instructions");
5529 Opc = AArch64::LDR_ZXI;
5534 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5535 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5536 Opc = AArch64::LD1Threev1d;
5541 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5542 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5543 Opc = AArch64::LD1Fourv1d;
5545 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5546 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5547 Opc = AArch64::LD1Twov2d;
5549 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5550 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5552 "Unexpected register load without SVE load instructions");
5553 Opc = AArch64::LDR_ZZXI;
5558 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5559 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5560 Opc = AArch64::LD1Threev2d;
5562 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5564 "Unexpected register load without SVE load instructions");
5565 Opc = AArch64::LDR_ZZZXI;
5570 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5571 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5572 Opc = AArch64::LD1Fourv2d;
5574 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5575 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5577 "Unexpected register load without SVE load instructions");
5578 Opc = AArch64::LDR_ZZZZXI;
5584 assert(Opc &&
"Unknown register class");
5594 MI.addMemOperand(MMO);
5601 UseMI.getIterator()),
5603 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5604 I.readsRegister(AArch64::NZCV, TRI);
5613 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5620 ByteSized =
Offset.getFixed();
5621 VGSized =
Offset.getScalable() / 2;
5629 int64_t &NumDataVectors) {
5633 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5635 NumBytes =
Offset.getFixed();
5637 NumPredicateVectors =
Offset.getScalable() / 2;
5642 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5643 NumPredicateVectors > 62) {
5644 NumDataVectors = NumPredicateVectors / 8;
5645 NumPredicateVectors -= NumDataVectors * 8;
5652 int NumVGScaledBytes,
unsigned VG,
5660 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5663 if (NumVGScaledBytes) {
5674 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5675 << std::abs(NumVGScaledBytes) <<
" * VG";
5684 int64_t NumBytes, NumVGScaledBytes;
5687 std::string CommentBuffer;
5690 if (Reg == AArch64::SP)
5692 else if (Reg == AArch64::FP)
5699 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5703 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5707 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
5716 unsigned FrameReg,
unsigned Reg,
5718 bool LastAdjustmentWasScalable) {
5719 if (
Offset.getScalable())
5722 if (FrameReg == Reg && !LastAdjustmentWasScalable)
5725 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5732 int64_t NumBytes, NumVGScaledBytes;
5734 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
5736 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5739 if (!NumVGScaledBytes)
5742 std::string CommentBuffer;
5749 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5753 CfaExpr.
push_back(dwarf::DW_CFA_expression);
5769 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
5772 bool *HasWinCFI,
bool EmitCFAOffset,
5775 unsigned MaxEncoding, ShiftSize;
5777 case AArch64::ADDXri:
5778 case AArch64::ADDSXri:
5779 case AArch64::SUBXri:
5780 case AArch64::SUBSXri:
5781 MaxEncoding = 0xfff;
5784 case AArch64::ADDVL_XXI:
5785 case AArch64::ADDPL_XXI:
5786 case AArch64::ADDSVL_XXI:
5787 case AArch64::ADDSPL_XXI:
5802 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
5804 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
5818 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
5820 if (TmpReg == AArch64::XZR)
5822 &AArch64::GPR64RegClass);
5824 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
5825 unsigned LocalShiftSize = 0;
5826 if (ThisVal > MaxEncoding) {
5827 ThisVal = ThisVal >> ShiftSize;
5828 LocalShiftSize = ShiftSize;
5830 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
5831 "Encoding cannot handle value that big");
5833 Offset -= ThisVal << LocalShiftSize;
5838 .
addImm(Sign * (
int)ThisVal);
5848 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
5849 CFAOffset += Change;
5851 CFAOffset -= Change;
5852 if (EmitCFAOffset && DestReg == TmpReg) {
5865 assert(Sign == 1 &&
"SEH directives should always have a positive sign");
5866 int Imm = (int)(ThisVal << LocalShiftSize);
5867 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
5868 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
5877 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
5878 "emit a single SEH directive");
5879 }
else if (DestReg == AArch64::SP) {
5882 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
5895 unsigned DestReg,
unsigned SrcReg,
5898 bool NeedsWinCFI,
bool *HasWinCFI,
5900 unsigned FrameReg) {
5907 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
5909 int64_t Bytes, NumPredicateVectors, NumDataVectors;
5911 Offset, Bytes, NumPredicateVectors, NumDataVectors);
5914 if (Bytes || (!
Offset && SrcReg != DestReg)) {
5915 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
5916 "SP increment/decrement not 8-byte aligned");
5917 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
5920 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
5923 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
5925 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
5932 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
5933 "SetNZCV not supported with SVE vectors");
5934 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
5935 "WinCFI not supported with SVE vectors");
5937 if (NumDataVectors) {
5939 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
5940 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5941 CFAOffset, FrameReg);
5946 if (NumPredicateVectors) {
5947 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
5949 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
5950 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5951 CFAOffset, FrameReg);
5969 if (
MI.isFullCopy()) {
5972 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
5976 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
5981 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
6009 if (
MI.isCopy() && Ops.
size() == 1 &&
6011 (Ops[0] == 0 || Ops[0] == 1)) {
6012 bool IsSpill = Ops[0] == 0;
6013 bool IsFill = !IsSpill;
6025 :
TRI.getMinimalPhysRegClass(Reg);
6031 "Mismatched register size in non subreg COPY");
6038 return &*--InsertPt;
6050 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
6053 "Unexpected subreg on physical register");
6055 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
6057 return &*--InsertPt;
6075 case AArch64::sub_32:
6076 FillRC = &AArch64::GPR32RegClass;
6079 FillRC = &AArch64::FPR32RegClass;
6082 FillRC = &AArch64::FPR64RegClass;
6088 TRI.getRegSizeInBits(*FillRC) &&
6089 "Mismatched regclass size on folded subreg COPY");
6108 bool *OutUseUnscaledOp,
6109 unsigned *OutUnscaledOp,
6110 int64_t *EmittableOffset) {
6112 if (EmittableOffset)
6113 *EmittableOffset = 0;
6114 if (OutUseUnscaledOp)
6115 *OutUseUnscaledOp =
false;
6121 switch (
MI.getOpcode()) {
6124 case AArch64::LD1Rv1d:
6125 case AArch64::LD1Rv2s:
6126 case AArch64::LD1Rv2d:
6127 case AArch64::LD1Rv4h:
6128 case AArch64::LD1Rv4s:
6129 case AArch64::LD1Rv8b:
6130 case AArch64::LD1Rv8h:
6131 case AArch64::LD1Rv16b:
6132 case AArch64::LD1Twov2d:
6133 case AArch64::LD1Threev2d:
6134 case AArch64::LD1Fourv2d:
6135 case AArch64::LD1Twov1d:
6136 case AArch64::LD1Threev1d:
6137 case AArch64::LD1Fourv1d:
6138 case AArch64::ST1Twov2d:
6139 case AArch64::ST1Threev2d:
6140 case AArch64::ST1Fourv2d:
6141 case AArch64::ST1Twov1d:
6142 case AArch64::ST1Threev1d:
6143 case AArch64::ST1Fourv1d:
6144 case AArch64::ST1i8:
6145 case AArch64::ST1i16:
6146 case AArch64::ST1i32:
6147 case AArch64::ST1i64:
6149 case AArch64::IRGstack:
6150 case AArch64::STGloop:
6151 case AArch64::STZGloop:
6156 TypeSize ScaleValue(0U,
false), Width(0U,
false);
6157 int64_t MinOff, MaxOff;
6163 bool IsMulVL = ScaleValue.isScalable();
6164 unsigned Scale = ScaleValue.getKnownMinValue();
6174 std::optional<unsigned> UnscaledOp =
6176 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
6177 if (useUnscaledOp &&
6182 Scale = ScaleValue.getKnownMinValue();
6183 assert(IsMulVL == ScaleValue.isScalable() &&
6184 "Unscaled opcode has different value for scalable");
6186 int64_t Remainder =
Offset % Scale;
6187 assert(!(Remainder && useUnscaledOp) &&
6188 "Cannot have remainder when using unscaled op");
6190 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
6191 int64_t NewOffset =
Offset / Scale;
6192 if (MinOff <= NewOffset && NewOffset <= MaxOff)
6195 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
6199 if (EmittableOffset)
6200 *EmittableOffset = NewOffset;
6201 if (OutUseUnscaledOp)
6202 *OutUseUnscaledOp = useUnscaledOp;
6203 if (OutUnscaledOp && UnscaledOp)
6204 *OutUnscaledOp = *UnscaledOp;
6217 unsigned Opcode =
MI.getOpcode();
6218 unsigned ImmIdx = FrameRegIdx + 1;
6220 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
6225 MI.eraseFromParent();
6231 unsigned UnscaledOp;
6234 &UnscaledOp, &NewOffset);
6238 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
6240 MI.setDesc(
TII->get(UnscaledOp));
6242 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
6265 case AArch64::ADDSWrr:
6266 case AArch64::ADDSWri:
6267 case AArch64::ADDSXrr:
6268 case AArch64::ADDSXri:
6269 case AArch64::SUBSWrr:
6270 case AArch64::SUBSXrr:
6272 case AArch64::SUBSWri:
6273 case AArch64::SUBSXri:
6284 case AArch64::ADDWrr:
6285 case AArch64::ADDWri:
6286 case AArch64::SUBWrr:
6287 case AArch64::ADDSWrr:
6288 case AArch64::ADDSWri:
6289 case AArch64::SUBSWrr:
6291 case AArch64::SUBWri:
6292 case AArch64::SUBSWri:
6303 case AArch64::ADDXrr:
6304 case AArch64::ADDXri:
6305 case AArch64::SUBXrr:
6306 case AArch64::ADDSXrr:
6307 case AArch64::ADDSXri:
6308 case AArch64::SUBSXrr:
6310 case AArch64::SUBXri:
6311 case AArch64::SUBSXri:
6312 case AArch64::ADDv8i8:
6313 case AArch64::ADDv16i8:
6314 case AArch64::ADDv4i16:
6315 case AArch64::ADDv8i16:
6316 case AArch64::ADDv2i32:
6317 case AArch64::ADDv4i32:
6318 case AArch64::SUBv8i8:
6319 case AArch64::SUBv16i8:
6320 case AArch64::SUBv4i16:
6321 case AArch64::SUBv8i16:
6322 case AArch64::SUBv2i32:
6323 case AArch64::SUBv4i32:
6336 case AArch64::FADDHrr:
6337 case AArch64::FADDSrr:
6338 case AArch64::FADDDrr:
6339 case AArch64::FADDv4f16:
6340 case AArch64::FADDv8f16:
6341 case AArch64::FADDv2f32:
6342 case AArch64::FADDv2f64:
6343 case AArch64::FADDv4f32:
6344 case AArch64::FSUBHrr:
6345 case AArch64::FSUBSrr:
6346 case AArch64::FSUBDrr:
6347 case AArch64::FSUBv4f16:
6348 case AArch64::FSUBv8f16:
6349 case AArch64::FSUBv2f32:
6350 case AArch64::FSUBv2f64:
6351 case AArch64::FSUBv4f32:
6355 return Options.UnsafeFPMath ||
6372 unsigned CombineOpc,
unsigned ZeroReg = 0,
6373 bool CheckZeroReg =
false) {
6380 if (!
MI ||
MI->getParent() != &
MBB || (
unsigned)
MI->getOpcode() != CombineOpc)
6383 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
6387 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
6388 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
6389 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
6391 if (
MI->getOperand(3).getReg() != ZeroReg)
6396 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
6405 unsigned MulOpc,
unsigned ZeroReg) {
6421 bool Invert)
const {
6427 case AArch64::FADDHrr:
6428 case AArch64::FADDSrr:
6429 case AArch64::FADDDrr:
6430 case AArch64::FMULHrr:
6431 case AArch64::FMULSrr:
6432 case AArch64::FMULDrr:
6433 case AArch64::FMULX16:
6434 case AArch64::FMULX32:
6435 case AArch64::FMULX64:
6437 case AArch64::FADDv4f16:
6438 case AArch64::FADDv8f16:
6439 case AArch64::FADDv2f32:
6440 case AArch64::FADDv4f32:
6441 case AArch64::FADDv2f64:
6442 case AArch64::FMULv4f16:
6443 case AArch64::FMULv8f16:
6444 case AArch64::FMULv2f32:
6445 case AArch64::FMULv4f32:
6446 case AArch64::FMULv2f64:
6447 case AArch64::FMULXv4f16:
6448 case AArch64::FMULXv8f16:
6449 case AArch64::FMULXv2f32:
6450 case AArch64::FMULXv4f32:
6451 case AArch64::FMULXv2f64:
6455 case AArch64::FADD_ZZZ_H:
6456 case AArch64::FADD_ZZZ_S:
6457 case AArch64::FADD_ZZZ_D:
6458 case AArch64::FMUL_ZZZ_H:
6459 case AArch64::FMUL_ZZZ_S:
6460 case AArch64::FMUL_ZZZ_D:
6472 case AArch64::ADDWrr:
6473 case AArch64::ADDXrr:
6474 case AArch64::ANDWrr:
6475 case AArch64::ANDXrr:
6476 case AArch64::ORRWrr:
6477 case AArch64::ORRXrr:
6478 case AArch64::EORWrr:
6479 case AArch64::EORXrr:
6480 case AArch64::EONWrr:
6481 case AArch64::EONXrr:
6485 case AArch64::ADDv8i8:
6486 case AArch64::ADDv16i8:
6487 case AArch64::ADDv4i16:
6488 case AArch64::ADDv8i16:
6489 case AArch64::ADDv2i32:
6490 case AArch64::ADDv4i32:
6491 case AArch64::ADDv1i64:
6492 case AArch64::ADDv2i64:
6493 case AArch64::MULv8i8:
6494 case AArch64::MULv16i8:
6495 case AArch64::MULv4i16:
6496 case AArch64::MULv8i16:
6497 case AArch64::MULv2i32:
6498 case AArch64::MULv4i32:
6499 case AArch64::ANDv8i8:
6500 case AArch64::ANDv16i8:
6501 case AArch64::ORRv8i8:
6502 case AArch64::ORRv16i8:
6503 case AArch64::EORv8i8:
6504 case AArch64::EORv16i8:
6506 case AArch64::ADD_ZZZ_B:
6507 case AArch64::ADD_ZZZ_H:
6508 case AArch64::ADD_ZZZ_S:
6509 case AArch64::ADD_ZZZ_D:
6510 case AArch64::MUL_ZZZ_B:
6511 case AArch64::MUL_ZZZ_H:
6512 case AArch64::MUL_ZZZ_S:
6513 case AArch64::MUL_ZZZ_D:
6514 case AArch64::AND_ZZZ:
6515 case AArch64::ORR_ZZZ:
6516 case AArch64::EOR_ZZZ:
6547 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6555 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6567 case AArch64::ADDWrr:
6569 "ADDWrr does not have register operands");
6570 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6571 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6573 case AArch64::ADDXrr:
6574 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6575 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6577 case AArch64::SUBWrr:
6578 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6579 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6581 case AArch64::SUBXrr:
6582 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6583 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6585 case AArch64::ADDWri:
6586 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6588 case AArch64::ADDXri:
6589 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6591 case AArch64::SUBWri:
6592 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6594 case AArch64::SUBXri:
6595 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6597 case AArch64::ADDv8i8:
6598 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6599 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6601 case AArch64::ADDv16i8:
6602 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6603 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6605 case AArch64::ADDv4i16:
6606 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6607 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6608 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6609 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6611 case AArch64::ADDv8i16:
6612 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6613 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6614 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6615 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6617 case AArch64::ADDv2i32:
6618 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6619 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6620 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6621 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6623 case AArch64::ADDv4i32:
6624 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6625 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6626 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6627 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6629 case AArch64::SUBv8i8:
6630 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6631 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6633 case AArch64::SUBv16i8:
6634 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6635 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6637 case AArch64::SUBv4i16:
6638 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6639 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
6640 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
6641 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
6643 case AArch64::SUBv8i16:
6644 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
6645 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
6646 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
6647 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
6649 case AArch64::SUBv2i32:
6650 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
6651 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
6652 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
6653 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
6655 case AArch64::SUBv4i32:
6656 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
6657 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
6658 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
6659 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
6676 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
6688 assert(
false &&
"Unsupported FP instruction in combiner\n");
6690 case AArch64::FADDHrr:
6692 "FADDHrr does not have register operands");
6694 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
6695 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
6697 case AArch64::FADDSrr:
6699 "FADDSrr does not have register operands");
6701 Found |=
Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
6702 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
6704 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
6705 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
6707 case AArch64::FADDDrr:
6708 Found |=
Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
6709 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
6711 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
6712 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
6714 case AArch64::FADDv4f16:
6715 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
6716 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
6718 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
6719 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
6721 case AArch64::FADDv8f16:
6722 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
6723 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
6725 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
6726 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
6728 case AArch64::FADDv2f32:
6729 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
6730 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
6732 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
6733 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
6735 case AArch64::FADDv2f64:
6736 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
6737 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
6739 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
6740 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
6742 case AArch64::FADDv4f32:
6743 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
6744 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
6746 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
6747 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
6749 case AArch64::FSUBHrr:
6750 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
6751 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
6752 Found |=
Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
6754 case AArch64::FSUBSrr:
6755 Found =
Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
6757 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
6758 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
6760 Found |=
Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
6762 case AArch64::FSUBDrr:
6763 Found =
Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
6765 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
6766 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
6768 Found |=
Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
6770 case AArch64::FSUBv4f16:
6771 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
6772 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
6774 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
6775 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
6777 case AArch64::FSUBv8f16:
6778 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
6779 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
6781 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
6782 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
6784 case AArch64::FSUBv2f32:
6785 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
6786 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
6788 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
6789 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
6791 case AArch64::FSUBv2f64:
6792 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
6793 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
6795 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
6796 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
6798 case AArch64::FSUBv4f32:
6799 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
6800 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
6802 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
6803 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
6814 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
6821 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
6822 MI->getOperand(1).getReg().isVirtual())
6823 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
6824 if (
MI &&
MI->getOpcode() == Opcode) {
6836 case AArch64::FMULv2f32:
6837 Found =
Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
6838 Found |=
Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
6840 case AArch64::FMULv2f64:
6841 Found =
Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
6842 Found |=
Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
6844 case AArch64::FMULv4f16:
6845 Found =
Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
6846 Found |=
Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
6848 case AArch64::FMULv4f32:
6849 Found =
Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
6850 Found |=
Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
6852 case AArch64::FMULv8f16:
6853 Found =
Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
6854 Found |=
Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
6867 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
6870 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
6871 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
6885 case AArch64::FNEGDr:
6887 case AArch64::FNEGSr:
7019 case AArch64::SUBWrr:
7020 case AArch64::SUBSWrr:
7021 case AArch64::SUBXrr:
7022 case AArch64::SUBSXrr:
7064 bool DoRegPressureReduce)
const {
7081 DoRegPressureReduce);
7110 const Register *ReplacedAddend =
nullptr) {
7111 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7113 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
7116 Register SrcReg0 = MUL->getOperand(1).getReg();
7117 bool Src0IsKill = MUL->getOperand(1).isKill();
7118 Register SrcReg1 = MUL->getOperand(2).getReg();
7119 bool Src1IsKill = MUL->getOperand(2).isKill();
7123 if (ReplacedAddend) {
7125 SrcReg2 = *ReplacedAddend;
7133 MRI.constrainRegClass(ResultReg, RC);
7135 MRI.constrainRegClass(SrcReg0, RC);
7137 MRI.constrainRegClass(SrcReg1, RC);
7139 MRI.constrainRegClass(SrcReg2, RC);
7142 if (kind == FMAInstKind::Default)
7147 else if (kind == FMAInstKind::Indexed)
7152 .
addImm(MUL->getOperand(3).getImm());
7153 else if (kind == FMAInstKind::Accumulator)
7159 assert(
false &&
"Invalid FMA instruction kind \n");
7173 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
7174 Opc = AArch64::FNMADDSrrr;
7175 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
7176 Opc = AArch64::FNMADDDrrr;
7188 MRI.constrainRegClass(ResultReg, RC);
7190 MRI.constrainRegClass(SrcReg0, RC);
7192 MRI.constrainRegClass(SrcReg1, RC);
7194 MRI.constrainRegClass(SrcReg2, RC);
7210 unsigned IdxDupOp,
unsigned MulOpc,
7212 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
7213 "Invalid index of FMUL operand");
7221 if (Dup->
getOpcode() == TargetOpcode::COPY)
7225 MRI.clearKillFlags(DupSrcReg);
7226 MRI.constrainRegClass(DupSrcReg, RC);
7230 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
7254 FMAInstKind::Accumulator);
7271 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7286 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
7288 FMAInstKind::Accumulator, &NewVR);
7300 FMAInstKind::Indexed);
7313 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
7316 FMAInstKind::Indexed, &NewVR);
7341 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
7343 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7347 Register SrcReg0 = MUL->getOperand(1).getReg();
7348 bool Src0IsKill = MUL->getOperand(1).isKill();
7349 Register SrcReg1 = MUL->getOperand(2).getReg();
7350 bool Src1IsKill = MUL->getOperand(2).isKill();
7353 MRI.constrainRegClass(ResultReg, RC);
7355 MRI.constrainRegClass(SrcReg0, RC);
7357 MRI.constrainRegClass(SrcReg1, RC);
7359 MRI.constrainRegClass(VR, RC);
7381 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
7382 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
7396 if (Opcode == AArch64::SUBSWrr)
7397 Opcode = AArch64::SUBWrr;
7398 else if (Opcode == AArch64::SUBSXrr)
7399 Opcode = AArch64::SUBXrr;
7401 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
7402 "Unexpected instruction opcode.");
7419 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7446 DelInstrs, InstrIdxForVirtReg);
7452 InstrIdxForVirtReg);
7458 InstrIdxForVirtReg);
7467 Opc = AArch64::MADDWrrr;
7468 RC = &AArch64::GPR32RegClass;
7470 Opc = AArch64::MADDXrrr;
7471 RC = &AArch64::GPR64RegClass;
7482 Opc = AArch64::MADDWrrr;
7483 RC = &AArch64::GPR32RegClass;
7485 Opc = AArch64::MADDXrrr;
7486 RC = &AArch64::GPR64RegClass;
7498 unsigned BitSize, OrrOpc, ZeroReg;
7500 OrrOpc = AArch64::ORRWri;
7501 OrrRC = &AArch64::GPR32spRegClass;
7503 ZeroReg = AArch64::WZR;
7504 Opc = AArch64::MADDWrrr;
7505 RC = &AArch64::GPR32RegClass;
7507 OrrOpc = AArch64::ORRXri;
7508 OrrRC = &AArch64::GPR64spRegClass;
7510 ZeroReg = AArch64::XZR;
7511 Opc = AArch64::MADDXrrr;
7512 RC = &AArch64::GPR64RegClass;
7514 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7525 if (
Insn.size() != 1)
7527 auto MovI =
Insn.begin();
7530 if (MovI->Opcode == OrrOpc)
7536 assert((MovI->Opcode == AArch64::MOVNWi ||
7537 MovI->Opcode == AArch64::MOVZWi) &&
7540 assert((MovI->Opcode == AArch64::MOVNXi ||
7541 MovI->Opcode == AArch64::MOVZXi) &&
7548 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7549 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7560 unsigned SubOpc, ZeroReg;
7562 SubOpc = AArch64::SUBWrr;
7563 SubRC = &AArch64::GPR32spRegClass;
7564 ZeroReg = AArch64::WZR;
7565 Opc = AArch64::MADDWrrr;
7566 RC = &AArch64::GPR32RegClass;
7568 SubOpc = AArch64::SUBXrr;
7569 SubRC = &AArch64::GPR64spRegClass;
7570 ZeroReg = AArch64::XZR;
7571 Opc = AArch64::MADDXrrr;
7572 RC = &AArch64::GPR64RegClass;
7574 Register NewVR =
MRI.createVirtualRegister(SubRC);
7581 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7582 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7592 Opc = AArch64::MSUBWrrr;
7593 RC = &AArch64::GPR32RegClass;
7595 Opc = AArch64::MSUBXrrr;
7596 RC = &AArch64::GPR64RegClass;
7608 unsigned BitSize, OrrOpc, ZeroReg;
7610 OrrOpc = AArch64::ORRWri;
7611 OrrRC = &AArch64::GPR32spRegClass;
7613 ZeroReg = AArch64::WZR;
7614 Opc = AArch64::MADDWrrr;
7615 RC = &AArch64::GPR32RegClass;
7617 OrrOpc = AArch64::ORRXri;
7618 OrrRC = &AArch64::GPR64spRegClass;
7620 ZeroReg = AArch64::XZR;
7621 Opc = AArch64::MADDXrrr;
7622 RC = &AArch64::GPR64RegClass;
7624 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7634 if (
Insn.size() != 1)
7636 auto MovI =
Insn.begin();
7639 if (MovI->Opcode == OrrOpc)
7645 assert((MovI->Opcode == AArch64::MOVNWi ||
7646 MovI->Opcode == AArch64::MOVZWi) &&
7649 assert((MovI->Opcode == AArch64::MOVNXi ||
7650 MovI->Opcode == AArch64::MOVZXi) &&
7657 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7658 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7663 Opc = AArch64::MLAv8i8;
7664 RC = &AArch64::FPR64RegClass;
7668 Opc = AArch64::MLAv8i8;
7669 RC = &AArch64::FPR64RegClass;
7673 Opc = AArch64::MLAv16i8;
7674 RC = &AArch64::FPR128RegClass;
7678 Opc = AArch64::MLAv16i8;
7679 RC = &AArch64::FPR128RegClass;
7683 Opc = AArch64::MLAv4i16;
7684 RC = &AArch64::FPR64RegClass;
7688 Opc = AArch64::MLAv4i16;
7689 RC = &AArch64::FPR64RegClass;
7693 Opc = AArch64::MLAv8i16;
7694 RC = &AArch64::FPR128RegClass;
7698 Opc = AArch64::MLAv8i16;
7699 RC = &AArch64::FPR128RegClass;
7703 Opc = AArch64::MLAv2i32;
7704 RC = &AArch64::FPR64RegClass;
7708 Opc = AArch64::MLAv2i32;
7709 RC = &AArch64::FPR64RegClass;
7713 Opc = AArch64::MLAv4i32;
7714 RC = &AArch64::FPR128RegClass;
7718 Opc = AArch64::MLAv4i32;
7719 RC = &AArch64::FPR128RegClass;
7724 Opc = AArch64::MLAv8i8;
7725 RC = &AArch64::FPR64RegClass;
7727 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
7731 Opc = AArch64::MLSv8i8;
7732 RC = &AArch64::FPR64RegClass;
7736 Opc = AArch64::MLAv16i8;
7737 RC = &AArch64::FPR128RegClass;
7739 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
7743 Opc = AArch64::MLSv16i8;
7744 RC = &AArch64::FPR128RegClass;
7748 Opc = AArch64::MLAv4i16;
7749 RC = &AArch64::FPR64RegClass;
7751 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7755 Opc = AArch64::MLSv4i16;
7756 RC = &AArch64::FPR64RegClass;
7760 Opc = AArch64::MLAv8i16;
7761 RC = &AArch64::FPR128RegClass;
7763 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7767 Opc = AArch64::MLSv8i16;
7768 RC = &AArch64::FPR128RegClass;
7772 Opc = AArch64::MLAv2i32;
7773 RC = &AArch64::FPR64RegClass;
7775 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7779 Opc = AArch64::MLSv2i32;
7780 RC = &AArch64::FPR64RegClass;
7784 Opc = AArch64::MLAv4i32;
7785 RC = &AArch64::FPR128RegClass;
7787 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7791 Opc = AArch64::MLSv4i32;
7792 RC = &AArch64::FPR128RegClass;
7797 Opc = AArch64::MLAv4i16_indexed;
7798 RC = &AArch64::FPR64RegClass;
7802 Opc = AArch64::MLAv4i16_indexed;
7803 RC = &AArch64::FPR64RegClass;
7807 Opc = AArch64::MLAv8i16_indexed;
7808 RC = &AArch64::FPR128RegClass;
7812 Opc = AArch64::MLAv8i16_indexed;
7813 RC = &AArch64::FPR128RegClass;
7817 Opc = AArch64::MLAv2i32_indexed;
7818 RC = &AArch64::FPR64RegClass;
7822 Opc = AArch64::MLAv2i32_indexed;
7823 RC = &AArch64::FPR64RegClass;
7827 Opc = AArch64::MLAv4i32_indexed;
7828 RC = &AArch64::FPR128RegClass;
7832 Opc = AArch64::MLAv4i32_indexed;
7833 RC = &AArch64::FPR128RegClass;
7838 Opc = AArch64::MLAv4i16_indexed;
7839 RC = &AArch64::FPR64RegClass;
7841 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7845 Opc = AArch64::MLSv4i16_indexed;
7846 RC = &AArch64::FPR64RegClass;
7850 Opc = AArch64::MLAv8i16_indexed;
7851 RC = &AArch64::FPR128RegClass;
7853 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7857 Opc = AArch64::MLSv8i16_indexed;
7858 RC = &AArch64::FPR128RegClass;
7862 Opc = AArch64::MLAv2i32_indexed;
7863 RC = &AArch64::FPR64RegClass;
7865 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7869 Opc = AArch64::MLSv2i32_indexed;
7870 RC = &AArch64::FPR64RegClass;
7874 Opc = AArch64::MLAv4i32_indexed;
7875 RC = &AArch64::FPR128RegClass;
7877 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7881 Opc = AArch64::MLSv4i32_indexed;
7882 RC = &AArch64::FPR128RegClass;
7888 Opc = AArch64::FMADDHrrr;
7889 RC = &AArch64::FPR16RegClass;
7893 Opc = AArch64::FMADDSrrr;
7894 RC = &AArch64::FPR32RegClass;
7898 Opc = AArch64::FMADDDrrr;
7899 RC = &AArch64::FPR64RegClass;
7904 Opc = AArch64::FMADDHrrr;
7905 RC = &AArch64::FPR16RegClass;
7909 Opc = AArch64::FMADDSrrr;
7910 RC = &AArch64::FPR32RegClass;
7914 Opc = AArch64::FMADDDrrr;
7915 RC = &AArch64::FPR64RegClass;
7920 Opc = AArch64::FMLAv1i32_indexed;
7921 RC = &AArch64::FPR32RegClass;
7923 FMAInstKind::Indexed);
7926 Opc = AArch64::FMLAv1i32_indexed;
7927 RC = &AArch64::FPR32RegClass;
7929 FMAInstKind::Indexed);
7933 Opc = AArch64::FMLAv1i64_indexed;
7934 RC = &AArch64::FPR64RegClass;
7936 FMAInstKind::Indexed);
7939 Opc = AArch64::FMLAv1i64_indexed;
7940 RC = &AArch64::FPR64RegClass;
7942 FMAInstKind::Indexed);
7946 RC = &AArch64::FPR64RegClass;
7947 Opc = AArch64::FMLAv4i16_indexed;
7949 FMAInstKind::Indexed);
7952 RC = &AArch64::FPR64RegClass;
7953 Opc = AArch64::FMLAv4f16;
7955 FMAInstKind::Accumulator);
7958 RC = &AArch64::FPR64RegClass;
7959 Opc = AArch64::FMLAv4i16_indexed;
7961 FMAInstKind::Indexed);
7964 RC = &AArch64::FPR64RegClass;
7965 Opc = AArch64::FMLAv4f16;
7967 FMAInstKind::Accumulator);
7972 RC = &AArch64::FPR64RegClass;
7974 Opc = AArch64::FMLAv2i32_indexed;
7976 FMAInstKind::Indexed);
7978 Opc = AArch64::FMLAv2f32;
7980 FMAInstKind::Accumulator);
7985 RC = &AArch64::FPR64RegClass;
7987 Opc = AArch64::FMLAv2i32_indexed;
7989 FMAInstKind::Indexed);
7991 Opc = AArch64::FMLAv2f32;
7993 FMAInstKind::Accumulator);
7998 RC = &AArch64::FPR128RegClass;
7999 Opc = AArch64::FMLAv8i16_indexed;
8001 FMAInstKind::Indexed);
8004 RC = &AArch64::FPR128RegClass;
8005 Opc = AArch64::FMLAv8f16;
8007 FMAInstKind::Accumulator);
8010 RC = &AArch64::FPR128RegClass;
8011 Opc = AArch64::FMLAv8i16_indexed;
8013 FMAInstKind::Indexed);
8016 RC = &AArch64::FPR128RegClass;
8017 Opc = AArch64::FMLAv8f16;
8019 FMAInstKind::Accumulator);
8024 RC = &AArch64::FPR128RegClass;
8026 Opc = AArch64::FMLAv2i64_indexed;
8028 FMAInstKind::Indexed);
8030 Opc = AArch64::FMLAv2f64;
8032 FMAInstKind::Accumulator);
8037 RC = &AArch64::FPR128RegClass;
8039 Opc = AArch64::FMLAv2i64_indexed;
8041 FMAInstKind::Indexed);
8043 Opc = AArch64::FMLAv2f64;
8045 FMAInstKind::Accumulator);
8051 RC = &AArch64::FPR128RegClass;
8053 Opc = AArch64::FMLAv4i32_indexed;
8055 FMAInstKind::Indexed);
8057 Opc = AArch64::FMLAv4f32;
8059 FMAInstKind::Accumulator);
8065 RC = &AArch64::FPR128RegClass;
8067 Opc = AArch64::FMLAv4i32_indexed;
8069 FMAInstKind::Indexed);
8071 Opc = AArch64::FMLAv4f32;
8073 FMAInstKind::Accumulator);
8078 Opc = AArch64::FNMSUBHrrr;
8079 RC = &AArch64::FPR16RegClass;
8083 Opc = AArch64::FNMSUBSrrr;
8084 RC = &AArch64::FPR32RegClass;
8088 Opc = AArch64::FNMSUBDrrr;
8089 RC = &AArch64::FPR64RegClass;
8094 Opc = AArch64::FNMADDHrrr;
8095 RC = &AArch64::FPR16RegClass;
8099 Opc = AArch64::FNMADDSrrr;
8100 RC = &AArch64::FPR32RegClass;
8104 Opc = AArch64::FNMADDDrrr;
8105 RC = &AArch64::FPR64RegClass;
8110 Opc = AArch64::FMSUBHrrr;
8111 RC = &AArch64::FPR16RegClass;
8115 Opc = AArch64::FMSUBSrrr;
8116 RC = &AArch64::FPR32RegClass;
8120 Opc = AArch64::FMSUBDrrr;
8121 RC = &AArch64::FPR64RegClass;
8126 Opc = AArch64::FMLSv1i32_indexed;
8127 RC = &AArch64::FPR32RegClass;
8129 FMAInstKind::Indexed);
8133 Opc = AArch64::FMLSv1i64_indexed;
8134 RC = &AArch64::FPR64RegClass;
8136 FMAInstKind::Indexed);
8141 RC = &AArch64::FPR64RegClass;
8147 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8149 Opc = AArch64::FMLAv4f16;
8151 FMAInstKind::Accumulator, &NewVR);
8153 Opc = AArch64::FMLAv4i16_indexed;
8155 FMAInstKind::Indexed, &NewVR);
8160 RC = &AArch64::FPR64RegClass;
8161 Opc = AArch64::FMLSv4f16;
8163 FMAInstKind::Accumulator);
8166 RC = &AArch64::FPR64RegClass;
8167 Opc = AArch64::FMLSv4i16_indexed;
8169 FMAInstKind::Indexed);
8174 RC = &AArch64::FPR64RegClass;
8176 Opc = AArch64::FMLSv2i32_indexed;
8178 FMAInstKind::Indexed);
8180 Opc = AArch64::FMLSv2f32;
8182 FMAInstKind::Accumulator);
8188 RC = &AArch64::FPR128RegClass;
8194 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8196 Opc = AArch64::FMLAv8f16;
8198 FMAInstKind::Accumulator, &NewVR);
8200 Opc = AArch64::FMLAv8i16_indexed;
8202 FMAInstKind::Indexed, &NewVR);
8207 RC = &AArch64::FPR128RegClass;
8208 Opc = AArch64::FMLSv8f16;
8210 FMAInstKind::Accumulator);
8213 RC = &AArch64::FPR128RegClass;
8214 Opc = AArch64::FMLSv8i16_indexed;
8216 FMAInstKind::Indexed);
8221 RC = &AArch64::FPR128RegClass;
8223 Opc = AArch64::FMLSv2i64_indexed;
8225 FMAInstKind::Indexed);
8227 Opc = AArch64::FMLSv2f64;
8229 FMAInstKind::Accumulator);
8235 RC = &AArch64::FPR128RegClass;
8237 Opc = AArch64::FMLSv4i32_indexed;
8239 FMAInstKind::Indexed);
8241 Opc = AArch64::FMLSv4f32;
8243 FMAInstKind::Accumulator);
8248 RC = &AArch64::FPR64RegClass;
8254 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8256 Opc = AArch64::FMLAv2i32_indexed;
8258 FMAInstKind::Indexed, &NewVR);
8260 Opc = AArch64::FMLAv2f32;
8262 FMAInstKind::Accumulator, &NewVR);
8268 RC = &AArch64::FPR128RegClass;
8274 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8276 Opc = AArch64::FMLAv4i32_indexed;
8278 FMAInstKind::Indexed, &NewVR);
8280 Opc = AArch64::FMLAv4f32;
8282 FMAInstKind::Accumulator, &NewVR);
8288 RC = &AArch64::FPR128RegClass;
8294 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8296 Opc = AArch64::FMLAv2i64_indexed;
8298 FMAInstKind::Indexed, &NewVR);
8300 Opc = AArch64::FMLAv2f64;
8302 FMAInstKind::Accumulator, &NewVR);
8312 &AArch64::FPR128RegClass,
MRI);
8321 &AArch64::FPR128RegClass,
MRI);
8330 &AArch64::FPR128_loRegClass,
MRI);
8339 &AArch64::FPR128RegClass,
MRI);
8348 &AArch64::FPR128_loRegClass,
MRI);
8367 for (
auto *
MI : InsInstrs)
8368 MI->setFlags(Flags);
8409 bool IsNegativeBranch =
false;
8410 bool IsTestAndBranch =
false;
8411 unsigned TargetBBInMI = 0;
8412 switch (
MI.getOpcode()) {
8421 case AArch64::CBNZW:
8422 case AArch64::CBNZX:
8424 IsNegativeBranch =
true;
8429 IsTestAndBranch =
true;
8431 case AArch64::TBNZW:
8432 case AArch64::TBNZX:
8434 IsNegativeBranch =
true;
8435 IsTestAndBranch =
true;
8441 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
8445 assert(
MI.getParent() &&
"Incomplete machine instruciton\n");
8458 if (!
MRI->hasOneNonDBGUse(CopyVReg))
8460 if (!
MRI->hasOneDef(CopyVReg))
8469 case AArch64::ANDWri:
8470 case AArch64::ANDXri: {
8471 if (IsTestAndBranch)
8475 if (!
MRI->hasOneNonDBGUse(VReg))
8489 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
8495 unsigned Opc = (Imm < 32)
8496 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
8497 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
8510 if (!Is32Bit && Imm < 32)
8512 MI.eraseFromParent();
8516 case AArch64::CSINCWr:
8517 case AArch64::CSINCXr: {
8537 if (IsNegativeBranch)
8540 MI.eraseFromParent();
8546std::pair<unsigned, unsigned>
8549 return std::make_pair(TF & Mask, TF & ~Mask);
8554 using namespace AArch64II;
8556 static const std::pair<unsigned, const char *> TargetFlags[] = {
8557 {MO_PAGE,
"aarch64-page"}, {MO_PAGEOFF,
"aarch64-pageoff"},
8558 {MO_G3,
"aarch64-g3"}, {MO_G2,
"aarch64-g2"},
8559 {MO_G1,
"aarch64-g1"}, {MO_G0,
"aarch64-g0"},
8560 {MO_HI12,
"aarch64-hi12"}};
8566 using namespace AArch64II;
8568 static const std::pair<unsigned, const char *> TargetFlags[] = {
8569 {MO_COFFSTUB,
"aarch64-coffstub"},
8570 {MO_GOT,
"aarch64-got"},
8571 {MO_NC,
"aarch64-nc"},
8572 {MO_S,
"aarch64-s"},
8573 {MO_TLS,
"aarch64-tls"},
8574 {MO_DLLIMPORT,
"aarch64-dllimport"},
8575 {MO_PREL,
"aarch64-prel"},
8576 {MO_TAGGED,
"aarch64-tagged"},
8577 {MO_ARM64EC_CALLMANGLE,
"aarch64-arm64ec-callmangle"},
8584 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8692 for (
unsigned Reg : AArch64::GPR64RegClass) {
8694 Reg != AArch64::LR &&
8695 Reg != AArch64::X16 &&
8696 Reg != AArch64::X17 &&
8697 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
8698 C.isAvailableInsideSeq(
Reg,
TRI))
8729 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
8732std::optional<std::unique_ptr<outliner::OutlinedFunction>>
8735 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
8736 unsigned MinRepeats)
const {
8737 unsigned SequenceSize = 0;
8738 for (
auto &
MI : RepeatedSequenceLocs[0])
8741 unsigned NumBytesToCreateFrame = 0;
8751 if (std::adjacent_find(
8752 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
8756 if (outliningCandidatesSigningScopeConsensus(a, b) &&
8757 outliningCandidatesSigningKeyConsensus(a, b) &&
8758 outliningCandidatesV8_3OpsConsensus(a, b)) {
8762 }) != RepeatedSequenceLocs.end()) {
8763 return std::nullopt;
8780 unsigned NumBytesToCheckLRInTCEpilogue = 0;
8781 if (RepeatedSequenceLocs[0]
8783 ->getInfo<AArch64FunctionInfo>()
8784 ->shouldSignReturnAddress(
true)) {
8786 NumBytesToCreateFrame += 8;
8789 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
8790 *RepeatedSequenceLocs[0].getMF());
8791 NumBytesToCheckLRInTCEpilogue =
8795 if (isTailCallReturnInst(RepeatedSequenceLocs[0].back()))
8796 SequenceSize += NumBytesToCheckLRInTCEpilogue;
8804 for (
auto &
MI :
C) {
8805 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
8806 switch (
MI.getOpcode()) {
8807 case AArch64::ADDXri:
8808 case AArch64::ADDWri:
8809 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8811 "Expected operand to be immediate");
8813 "Expected operand to be a register");
8817 if (
MI.getOperand(1).getReg() == AArch64::SP)
8818 SPValue +=
MI.getOperand(2).getImm();
8822 case AArch64::SUBXri:
8823 case AArch64::SUBWri:
8824 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8826 "Expected operand to be immediate");
8828 "Expected operand to be a register");
8832 if (
MI.getOperand(1).getReg() == AArch64::SP)
8833 SPValue -=
MI.getOperand(2).getImm();
8850 if (RepeatedSequenceLocs.size() < MinRepeats)
8851 return std::nullopt;
8855 unsigned FlagsSetInAll = 0xF;
8859 FlagsSetInAll &=
C.Flags;
8861 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
8864 auto SetCandidateCallInfo =
8865 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
8867 C.setCallInfo(CallID, NumBytesForCall);
8871 NumBytesToCreateFrame += 4;
8874 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
8879 unsigned CFICount = 0;
8880 for (
auto &
I : RepeatedSequenceLocs[0]) {
8881 if (
I.isCFIInstruction())
8891 std::vector<MCCFIInstruction> CFIInstructions =
8892 C.getMF()->getFrameInstructions();
8894 if (CFICount > 0 && CFICount != CFIInstructions.size())
8895 return std::nullopt;
8903 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
8904 !
MI.readsRegister(AArch64::SP, &
TRI))
8910 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
8915 if (
MI.mayLoadOrStore()) {
8918 bool OffsetIsScalable;
8922 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
8923 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
8927 if (OffsetIsScalable)
8935 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
8936 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
8939 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
8940 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
8955 bool AllStackInstrsSafe =
8960 if (RepeatedSequenceLocs[0].back().isTerminator()) {
8962 NumBytesToCreateFrame = 0;
8963 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
8967 else if (LastInstrOpcode == AArch64::BL ||
8968 ((LastInstrOpcode == AArch64::BLR ||
8969 LastInstrOpcode == AArch64::BLRNoIP) &&
8973 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
8981 unsigned NumBytesNoStackCalls = 0;
8982 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
8987 (
C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
8988 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
8997 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
9000 if (LRAvailable && !IsNoReturn) {
9001 NumBytesNoStackCalls += 4;
9003 CandidatesWithoutStackFixups.push_back(
C);
9008 else if (findRegisterToSaveLRTo(
C)) {
9009 NumBytesNoStackCalls += 12;
9011 CandidatesWithoutStackFixups.push_back(
C);
9016 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
9017 NumBytesNoStackCalls += 12;
9019 CandidatesWithoutStackFixups.push_back(
C);
9025 NumBytesNoStackCalls += SequenceSize;
9032 if (!AllStackInstrsSafe ||
9033 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
9034 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
9036 if (RepeatedSequenceLocs.size() < MinRepeats)
9037 return std::nullopt;
9086 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
9090 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
9091 !findRegisterToSaveLRTo(
C));
9097 if (RepeatedSequenceLocs.size() < MinRepeats)
9098 return std::nullopt;
9103 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
9107 bool ModStackToSaveLR =
false;
9110 ModStackToSaveLR =
true;
9119 ModStackToSaveLR =
true;
9121 if (ModStackToSaveLR) {
9123 if (!AllStackInstrsSafe)
9124 return std::nullopt;
9127 NumBytesToCreateFrame += 8;
9134 return std::nullopt;
9136 return std::make_unique<outliner::OutlinedFunction>(
9137 RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID);
9141 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
9145 const auto &CFn = Candidates.front().getMF()->getFunction();
9147 if (CFn.hasFnAttribute(
"ptrauth-returns"))
9148 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-returns"));
9149 if (CFn.hasFnAttribute(
"ptrauth-auth-traps"))
9150 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-auth-traps"));
9153 if (CFn.hasFnAttribute(
"sign-return-address"))
9154 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
9155 if (CFn.hasFnAttribute(
"sign-return-address-key"))
9156 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
9158 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
9166 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
9180 if (!AFI || AFI->
hasRedZone().value_or(
true))
9200 unsigned &Flags)
const {
9202 "Must track liveness!");
9204 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
9219 auto AreAllUnsafeRegsDead = [&LRU]() {
9236 bool LRAvailableEverywhere =
true;
9241 if (
MI.isCall() && !
MI.isTerminator())
9242 Flags |= MachineOutlinerMBBFlags::HasCalls;
9247 auto CreateNewRangeStartingAt =
9248 [&RangeBegin, &RangeEnd,
9250 RangeBegin = NewBegin;
9251 RangeEnd = std::next(RangeBegin);
9254 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
9259 Ranges.push_back(std::make_pair(RangeBegin, RangeEnd));
9267 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
9271 UpdateWholeMBBFlags(*FirstPossibleEndPt);
9272 if (AreAllUnsafeRegsDead())
9279 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
9285 UpdateWholeMBBFlags(
MI);
9286 if (!AreAllUnsafeRegsDead()) {
9287 SaveRangeIfNonEmpty();
9288 CreateNewRangeStartingAt(
MI.getIterator());
9291 LRAvailableEverywhere &= LRU.
available(AArch64::LR);
9292 RangeBegin =
MI.getIterator();
9297 if (AreAllUnsafeRegsDead())
9298 SaveRangeIfNonEmpty();
9303 std::reverse(Ranges.begin(), Ranges.end());
9306 if (!LRAvailableEverywhere)
9307 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
9314 unsigned Flags)
const {
9322 switch (
MI.getOpcode()) {
9324 case AArch64::PACIASP:
9325 case AArch64::PACIBSP:
9326 case AArch64::PACIASPPC:
9327 case AArch64::PACIBSPPC:
9328 case AArch64::AUTIASP:
9329 case AArch64::AUTIBSP:
9330 case AArch64::AUTIASPPCi:
9331 case AArch64::AUTIASPPCr:
9332 case AArch64::AUTIBSPPCi:
9333 case AArch64::AUTIBSPPCr:
9334 case AArch64::RETAA:
9335 case AArch64::RETAB:
9336 case AArch64::RETAASPPCi:
9337 case AArch64::RETAASPPCr:
9338 case AArch64::RETABSPPCi:
9339 case AArch64::RETABSPPCr:
9340 case AArch64::EMITBKEY:
9341 case AArch64::PAUTH_PROLOGUE:
9342 case AArch64::PAUTH_EPILOGUE:
9356 if (
MI.isCFIInstruction())
9360 if (
MI.isTerminator())
9369 assert(!MOP.isCFIIndex());
9372 if (MOP.isReg() && !MOP.isImplicit() &&
9373 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
9380 if (
MI.getOpcode() == AArch64::ADRP)
9401 if (MOP.isGlobal()) {
9402 Callee = dyn_cast<Function>(MOP.getGlobal());
9409 if (Callee && Callee->getName() ==
"\01_mcount")
9417 if (
MI.getOpcode() == AArch64::BLR ||
9418 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
9422 return UnknownCallOutlineType;
9430 return UnknownCallOutlineType;
9438 return UnknownCallOutlineType;
9463 bool OffsetIsScalable;
9466 if (!
MI.mayLoadOrStore() ||
9469 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
9474 int64_t Dummy1, Dummy2;
9477 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
9479 assert(Scale != 0 &&
"Unexpected opcode!");
9480 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
9485 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
9486 StackOffsetOperand.
setImm(NewImm);
9492 bool ShouldSignReturnAddr) {
9493 if (!ShouldSignReturnAddr)
9499 TII->get(AArch64::PAUTH_EPILOGUE))
9515 unsigned TailOpcode;
9516 if (Call->getOpcode() == AArch64::BL) {
9517 TailOpcode = AArch64::TCRETURNdi;
9519 assert(Call->getOpcode() == AArch64::BLR ||
9520 Call->getOpcode() == AArch64::BLRNoIP);
9521 TailOpcode = AArch64::TCRETURNriALL;
9524 .
add(Call->getOperand(0))
9527 Call->eraseFromParent();
9532 bool IsLeafFunction =
true;
9536 return MI.isCall() && !
MI.isReturn();
9546 "Can only fix up stack references once");
9547 fixupPostOutline(
MBB);
9549 IsLeafFunction =
false;
9560 Et = std::prev(
MBB.
end());
9573 unsigned DwarfReg =
MRI->getDwarfRegNum(AArch64::LR,
true);
9576 int64_t StackPosEntry =
9629 fixupPostOutline(
MBB);
9640 .addGlobalAddress(M.getNamedValue(MF.
getName()))
9650 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9665 Register Reg = findRegisterToSaveLRTo(
C);
9666 assert(Reg &&
"No callee-saved register available?");
9700 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9716 bool AllowSideEffects)
const {
9721 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
9734 assert(STI.hasNEON() &&
"Expected to have NEON.");
9735 Register Reg64 =
TRI.getSubReg(Reg, AArch64::dsub);
9740std::optional<DestSourcePair>
9745 if (
MI.getOpcode() == AArch64::ORRWrs &&
9746 MI.getOperand(1).getReg() == AArch64::WZR &&
9747 MI.getOperand(3).getImm() == 0x0 &&
9749 (!
MI.getOperand(0).getReg().isVirtual() ||
9750 MI.getOperand(0).getSubReg() == 0) &&
9751 (!
MI.getOperand(0).getReg().isPhysical() ||
9752 MI.findRegisterDefOperandIdx(
MI.getOperand(0).getReg() - AArch64::W0 +
9757 if (
MI.getOpcode() == AArch64::ORRXrs &&
9758 MI.getOperand(1).getReg() == AArch64::XZR &&
9759 MI.getOperand(3).getImm() == 0x0)
9762 return std::nullopt;
9765std::optional<DestSourcePair>
9767 if (
MI.getOpcode() == AArch64::ORRWrs &&
9768 MI.getOperand(1).getReg() == AArch64::WZR &&
9769 MI.getOperand(3).getImm() == 0x0)
9771 return std::nullopt;
9774std::optional<RegImmPair>
9783 return std::nullopt;
9785 switch (
MI.getOpcode()) {
9787 return std::nullopt;
9788 case AArch64::SUBWri:
9789 case AArch64::SUBXri:
9790 case AArch64::SUBSWri:
9791 case AArch64::SUBSXri:
9794 case AArch64::ADDSWri:
9795 case AArch64::ADDSXri:
9796 case AArch64::ADDWri:
9797 case AArch64::ADDXri: {
9799 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
9800 !
MI.getOperand(2).isImm())
9801 return std::nullopt;
9802 int Shift =
MI.getOperand(3).getImm();
9803 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
9804 Offset = Sign * (
MI.getOperand(2).getImm() << Shift);
9813static std::optional<ParamLoadedValue>
9817 auto DestSrc =
TII->isCopyLikeInstr(
MI);
9819 return std::nullopt;
9821 Register DestReg = DestSrc->Destination->getReg();
9822 Register SrcReg = DestSrc->Source->getReg();
9827 if (DestReg == DescribedReg)
9831 if (
MI.getOpcode() == AArch64::ORRWrs &&
9832 TRI->isSuperRegister(DestReg, DescribedReg))
9836 if (
MI.getOpcode() == AArch64::ORRXrs &&
9837 TRI->isSubRegister(DestReg, DescribedReg)) {
9838 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
9842 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
9843 "Unhandled ORR[XW]rs copy case");
9845 return std::nullopt;
9864 return MI.getOpcode() == AArch64::INLINEASM_BR;
9882 switch (
MI.getOpcode()) {
9883 case TargetOpcode::G_BRJT:
9884 case AArch64::JumpTableDest32:
9885 case AArch64::JumpTableDest16:
9886 case AArch64::JumpTableDest8:
9897std::optional<ParamLoadedValue>
9902 switch (
MI.getOpcode()) {
9903 case AArch64::MOVZWi:
9904 case AArch64::MOVZXi: {
9907 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
9908 return std::nullopt;
9910 if (!
MI.getOperand(1).isImm())
9911 return std::nullopt;
9912 int64_t Immediate =
MI.getOperand(1).getImm();
9913 int Shift =
MI.getOperand(2).getImm();
9917 case AArch64::ORRWrs:
9918 case AArch64::ORRXrs:
9928 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
9929 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
9932 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
9936 if (!
MRI.hasOneNonDBGUse(DefReg))
9941 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
9942 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
9963 unsigned Scale)
const {
9974 unsigned Shift =
Log2_64(NumBytes);
9975 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
9983 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
9988 return AArch64::BLRNoIP;
9990 return AArch64::BLR;
9995 Register TargetReg,
bool FrameSetup)
const {
9996 assert(TargetReg != AArch64::SP &&
"New top of stack cannot aleady be in SP");
10008 MF.
insert(MBBInsertPoint, LoopTestMBB);
10011 MF.
insert(MBBInsertPoint, LoopBodyMBB);
10013 MF.
insert(MBBInsertPoint, ExitMBB);
10023 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
10031 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::Bcc))
10037 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
10050 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
10075 return ExitMBB->
begin();
10092 unsigned CompCounterOprNum;
10096 unsigned UpdateCounterOprNum;
10100 bool IsUpdatePriorComp;
10112 TII(MF->getSubtarget().getInstrInfo()),
10113 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
10114 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
10115 CompCounterOprNum(CompCounterOprNum), Update(Update),
10116 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
10117 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
10119 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
10125 std::optional<bool> createTripCountGreaterCondition(
10135 void createRemainingIterationsGreaterCondition(
10141 void adjustTripCount(
int TripCountAdjust)
override {}
10143 void disposed()
override {}
10144 bool isMVEExpanderSupported()
override {
return true; }
10162 Result =
MRI.createVirtualRegister(
10165 }
else if (
I == ReplaceOprNum) {
10166 MRI.constrainRegClass(
10176void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
10192 assert(CondBranch->getOpcode() == AArch64::Bcc);
10196 if (CondBranch->getOperand(1).getMBB() == LoopBB)
10203 auto AccumulateCond = [&](
Register CurCond,
10205 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
10214 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
10218 for (
int I = 0;
I <= TC; ++
I) {
10224 AccCond = AccumulateCond(AccCond,
CC);
10228 if (Update != Comp && IsUpdatePriorComp) {
10230 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
10231 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
10235 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
10237 }
else if (Update != Comp) {
10242 Counter = NextCounter;
10246 if (LastStage0Insts.
empty()) {
10250 if (IsUpdatePriorComp)
10255 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
10258 for (
int I = 0;
I <= TC; ++
I) {
10262 AccCond = AccumulateCond(AccCond,
CC);
10263 if (
I != TC && Update != Comp)
10266 Counter = NextCounter;
10282 assert(Phi.getNumOperands() == 5);
10283 if (Phi.getOperand(2).getMBB() ==
MBB) {
10284 RegMBB = Phi.getOperand(1).getReg();
10285 RegOther = Phi.getOperand(3).getReg();
10287 assert(Phi.getOperand(4).getMBB() ==
MBB);
10288 RegMBB = Phi.getOperand(3).getReg();
10289 RegOther = Phi.getOperand(1).getReg();
10294 if (!Reg.isVirtual())
10297 return MRI.getVRegDef(Reg)->getParent() != BB;
10303 unsigned &UpdateCounterOprNum,
Register &InitReg,
10304 bool &IsUpdatePriorComp) {
10318 if (!Reg.isVirtual())
10321 UpdateInst =
nullptr;
10322 UpdateCounterOprNum = 0;
10324 IsUpdatePriorComp =
true;
10328 if (Def->getParent() != LoopBB)
10330 if (Def->isCopy()) {
10332 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
10334 CurReg = Def->getOperand(1).getReg();
10335 }
else if (Def->isPHI()) {
10339 IsUpdatePriorComp =
false;
10344 switch (Def->getOpcode()) {
10345 case AArch64::ADDSXri:
10346 case AArch64::ADDSWri:
10347 case AArch64::SUBSXri:
10348 case AArch64::SUBSWri:
10349 case AArch64::ADDXri:
10350 case AArch64::ADDWri:
10351 case AArch64::SUBXri:
10352 case AArch64::SUBWri:
10354 UpdateCounterOprNum = 1;
10356 case AArch64::ADDSXrr:
10357 case AArch64::ADDSWrr:
10358 case AArch64::SUBSXrr:
10359 case AArch64::SUBSWrr:
10360 case AArch64::ADDXrr:
10361 case AArch64::ADDWrr:
10362 case AArch64::SUBXrr:
10363 case AArch64::SUBWrr:
10366 UpdateCounterOprNum = 1;
10368 UpdateCounterOprNum = 2;
10375 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
10390std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
10401 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
10412 if (
TBB == LoopBB && FBB == LoopBB)
10416 if (
TBB != LoopBB && FBB ==
nullptr)
10419 assert((
TBB == LoopBB || FBB == LoopBB) &&
10420 "The Loop must be a single-basic-block loop");
10425 if (CondBranch->
getOpcode() != AArch64::Bcc)
10433 unsigned CompCounterOprNum = 0;
10435 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
10439 switch (
MI.getOpcode()) {
10440 case AArch64::SUBSXri:
10441 case AArch64::SUBSWri:
10442 case AArch64::ADDSXri:
10443 case AArch64::ADDSWri:
10445 CompCounterOprNum = 1;
10447 case AArch64::ADDSWrr:
10448 case AArch64::ADDSXrr:
10449 case AArch64::SUBSWrr:
10450 case AArch64::SUBSXrr:
10461 if (CompCounterOprNum == 0) {
10463 CompCounterOprNum = 2;
10465 CompCounterOprNum = 1;
10477 bool IsUpdatePriorComp;
10478 unsigned UpdateCounterOprNum;
10480 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
10483 return std::make_unique<AArch64PipelinerLoopInfo>(
10484 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
10495 TypeSize Scale(0U,
false), Width(0U,
false);
10496 int64_t MinOffset, MaxOffset;
10497 if (
getMemOpInfo(
MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
10499 if (
MI.getOperand(ImmIdx).isImm() && !
MI.getOperand(ImmIdx - 1).isFI()) {
10500 int64_t Imm =
MI.getOperand(ImmIdx).getImm();
10501 if (Imm < MinOffset || Imm > MaxOffset) {
10502 ErrInfo =
"Unexpected immediate on load/store instruction";
10510#define GET_INSTRINFO_HELPERS
10511#define GET_INSTRMAP_INFO
10512#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, MCRegister Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static bool isPostIndexLdStOpcode(unsigned Opcode)
Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
static unsigned getBranchDisplacementBits(unsigned Opc)
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallVector class.
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setOutliningStyle(const std::string &Style)
const SetOfInstructions & getLOHRelated() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors, int64_t &NumDataVectors)
Returns the offset in parts to which this frame offset can be decomposed for the purpose of describin...
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
uint64_t getElementSizeForOpcode(unsigned Opc) const
Returns the vector element size (B, H, S or D) of an SVE opcode.
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
bool isWhileOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE WHILE## instruction.
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
SmallVector< std::pair< MachineBasicBlock::iterator, MachineBasicBlock::iterator > > getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool useMachineCombiner() const override
AArch64 supports MachineCombiner.
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const override
static bool isFalkorShiftExtFast(const MachineInstr &MI)
Returns true if the instruction has a shift by immediate that can be executed in one cycle less.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
bool expandPostRAPseudo(MachineInstr &MI) const override
unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
bool isFunctionSafeToSplit(const MachineFunction &MF) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
Return true when Inst is associative and commutative so that it can be reassociated.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset, unsigned Scale) const
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
ArrayRef< std::pair< unsigned, const char * > > getSerializableBitmaskMachineOperandTargetFlags() const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isPTestLikeOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE instruction that sets the condition codes as if it's results...
void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const override
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isTargetILP32() const
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
AArch64PAuth::AuthCheckMethod getAuthenticatedLRCheckMethod(const MachineFunction &MF) const
Choose a method of checking LR before performing a tail call.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MBBSectionID getSectionID() const
Returns the section ID of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
Primary interface to the complete machine description for the target machine.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool succeeded(LogicalResult Result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
static bool isIndirectBranchOpcode(int Opc)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
unsigned getUndefRegState(bool B)
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
DWARFExpression::Operation Op
static bool isUncondBranchOpcode(int Opc)
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
static const MachineMemOperand::Flags MOStridedAccess
@ Default
The result values are uniform if and only if all operands are uniform.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
static const MBBSectionID ColdSectionID
MachineJumpTableEntry - One jump table in the jump table info.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.