62#define GET_INSTRINFO_CTOR_DTOR
63#include "AArch64GenInstrInfo.inc"
67 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
71 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
75 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
79 cl::desc(
"Restrict range of B instructions (DEBUG)"));
84 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
95 auto Op =
MI.getOpcode();
96 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
97 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
101 if (
MI.isMetaInstruction())
106 unsigned NumBytes = 0;
110 NumBytes =
Desc.getSize() ?
Desc.getSize() : 4;
113 if (!MFI->shouldSignReturnAddress(MF))
125 switch (
Desc.getOpcode()) {
128 return Desc.getSize();
135 case TargetOpcode::STACKMAP:
138 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
140 case TargetOpcode::PATCHPOINT:
143 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
145 case TargetOpcode::STATEPOINT:
147 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
152 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
157 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
159 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
160 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
164 case TargetOpcode::PATCHABLE_EVENT_CALL:
170 NumBytes =
MI.getOperand(1).getImm();
172 case TargetOpcode::BUNDLE:
173 NumBytes = getInstBundleLength(
MI);
180unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
184 while (++
I != E &&
I->isInsideBundle()) {
185 assert(!
I->isBundle() &&
"No nested bundle!");
244 int64_t BrOffset)
const {
246 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
247 "over conditional branch expansion");
248 return isIntN(Bits, BrOffset / 4);
253 switch (
MI.getOpcode()) {
257 return MI.getOperand(0).getMBB();
262 return MI.getOperand(2).getMBB();
268 return MI.getOperand(1).getMBB();
278 assert(RS &&
"RegScavenger required for long branching");
280 "new block should be inserted for expanding unconditional branch");
283 "restore block should be inserted for restoring clobbered registers");
288 if (!isInt<33>(BrOffset))
290 "Branch offsets outside of the signed 33-bit range not supported");
304 constexpr Register Reg = AArch64::X16;
306 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
314 if (Scavenged != AArch64::NoRegister &&
316 buildIndirectBranch(Scavenged, NewDestBB);
326 "Unable to insert indirect branch inside function that has red zone");
349 bool AllowModify)
const {
356 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
357 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
361 if (!isUnpredicatedTerminator(*
I))
368 unsigned LastOpc = LastInst->
getOpcode();
369 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
384 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
391 LastInst = SecondLastInst;
393 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
398 SecondLastInst = &*
I;
399 SecondLastOpc = SecondLastInst->
getOpcode();
410 LastInst = SecondLastInst;
412 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
414 "unreachable unconditional branches removed above");
423 SecondLastInst = &*
I;
424 SecondLastOpc = SecondLastInst->
getOpcode();
428 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
444 I->eraseFromParent();
453 I->eraseFromParent();
462 MachineBranchPredicate &MBP,
463 bool AllowModify)
const {
473 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
474 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
478 if (!isUnpredicatedTerminator(*
I))
483 unsigned LastOpc = LastInst->
getOpcode();
498 assert(MBP.TrueDest &&
"expected!");
501 MBP.ConditionDef =
nullptr;
502 MBP.SingleUseCondition =
false;
506 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
507 : MachineBranchPredicate::PRED_EQ;
513 if (
Cond[0].getImm() != -1) {
519 switch (
Cond[1].getImm()) {
523 Cond[1].setImm(AArch64::CBNZW);
526 Cond[1].setImm(AArch64::CBZW);
529 Cond[1].setImm(AArch64::CBNZX);
532 Cond[1].setImm(AArch64::CBZX);
535 Cond[1].setImm(AArch64::TBNZW);
538 Cond[1].setImm(AArch64::TBZW);
541 Cond[1].setImm(AArch64::TBNZX);
544 Cond[1].setImm(AArch64::TBZX);
553 int *BytesRemoved)
const {
563 I->eraseFromParent();
580 I->eraseFromParent();
587void AArch64InstrInfo::instantiateCondBranch(
590 if (
Cond[0].getImm() != -1) {
608 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
647 unsigned *NewVReg =
nullptr) {
652 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
655 unsigned SrcOpNum = 0;
657 case AArch64::ADDSXri:
658 case AArch64::ADDSWri:
665 case AArch64::ADDXri:
666 case AArch64::ADDWri:
672 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
675 case AArch64::ORNXrr:
676 case AArch64::ORNWrr: {
679 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
682 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
686 case AArch64::SUBSXrr:
687 case AArch64::SUBSWrr:
694 case AArch64::SUBXrr:
695 case AArch64::SUBWrr: {
698 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
701 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
707 assert(Opc && SrcOpNum &&
"Missing parameters");
719 int &FalseCycles)
const {
723 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
730 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
734 unsigned ExtraCondLat =
Cond.size() != 1;
738 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
739 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
741 CondCycles = 1 + ExtraCondLat;
742 TrueCycles = FalseCycles = 1;
752 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
753 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
754 CondCycles = 5 + ExtraCondLat;
755 TrueCycles = FalseCycles = 2;
772 switch (
Cond.size()) {
781 switch (
Cond[1].getImm()) {
804 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
810 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
820 switch (
Cond[1].getImm()) {
833 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
849 bool TryFold =
false;
850 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
851 RC = &AArch64::GPR64RegClass;
852 Opc = AArch64::CSELXr;
854 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
855 RC = &AArch64::GPR32RegClass;
856 Opc = AArch64::CSELWr;
858 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
859 RC = &AArch64::FPR64RegClass;
860 Opc = AArch64::FCSELDrrr;
861 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
862 RC = &AArch64::FPR32RegClass;
863 Opc = AArch64::FCSELSrrr;
865 assert(RC &&
"Unsupported regclass");
869 unsigned NewVReg = 0;
884 MRI.clearKillFlags(NewVReg);
889 MRI.constrainRegClass(TrueReg, RC);
890 MRI.constrainRegClass(FalseReg, RC);
905 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
910 return Is.
size() <= 2;
916 if (Subtarget.hasExynosCheapAsMoveHandling()) {
917 if (isExynosCheapAsMove(
MI))
919 return MI.isAsCheapAsAMove();
922 switch (
MI.getOpcode()) {
924 return MI.isAsCheapAsAMove();
926 case AArch64::ADDWrs:
927 case AArch64::ADDXrs:
928 case AArch64::SUBWrs:
929 case AArch64::SUBXrs:
930 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
935 case AArch64::MOVi32imm:
937 case AArch64::MOVi64imm:
943 switch (
MI.getOpcode()) {
947 case AArch64::ADDWrs:
948 case AArch64::ADDXrs:
949 case AArch64::ADDSWrs:
950 case AArch64::ADDSXrs: {
951 unsigned Imm =
MI.getOperand(3).getImm();
958 case AArch64::ADDWrx:
959 case AArch64::ADDXrx:
960 case AArch64::ADDXrx64:
961 case AArch64::ADDSWrx:
962 case AArch64::ADDSXrx:
963 case AArch64::ADDSXrx64: {
964 unsigned Imm =
MI.getOperand(3).getImm();
976 case AArch64::SUBWrs:
977 case AArch64::SUBSWrs: {
978 unsigned Imm =
MI.getOperand(3).getImm();
980 return ShiftVal == 0 ||
984 case AArch64::SUBXrs:
985 case AArch64::SUBSXrs: {
986 unsigned Imm =
MI.getOperand(3).getImm();
988 return ShiftVal == 0 ||
992 case AArch64::SUBWrx:
993 case AArch64::SUBXrx:
994 case AArch64::SUBXrx64:
995 case AArch64::SUBSWrx:
996 case AArch64::SUBSXrx:
997 case AArch64::SUBSXrx64: {
998 unsigned Imm =
MI.getOperand(3).getImm();
1010 case AArch64::LDRBBroW:
1011 case AArch64::LDRBBroX:
1012 case AArch64::LDRBroW:
1013 case AArch64::LDRBroX:
1014 case AArch64::LDRDroW:
1015 case AArch64::LDRDroX:
1016 case AArch64::LDRHHroW:
1017 case AArch64::LDRHHroX:
1018 case AArch64::LDRHroW:
1019 case AArch64::LDRHroX:
1020 case AArch64::LDRQroW:
1021 case AArch64::LDRQroX:
1022 case AArch64::LDRSBWroW:
1023 case AArch64::LDRSBWroX:
1024 case AArch64::LDRSBXroW:
1025 case AArch64::LDRSBXroX:
1026 case AArch64::LDRSHWroW:
1027 case AArch64::LDRSHWroX:
1028 case AArch64::LDRSHXroW:
1029 case AArch64::LDRSHXroX:
1030 case AArch64::LDRSWroW:
1031 case AArch64::LDRSWroX:
1032 case AArch64::LDRSroW:
1033 case AArch64::LDRSroX:
1034 case AArch64::LDRWroW:
1035 case AArch64::LDRWroX:
1036 case AArch64::LDRXroW:
1037 case AArch64::LDRXroX:
1038 case AArch64::PRFMroW:
1039 case AArch64::PRFMroX:
1040 case AArch64::STRBBroW:
1041 case AArch64::STRBBroX:
1042 case AArch64::STRBroW:
1043 case AArch64::STRBroX:
1044 case AArch64::STRDroW:
1045 case AArch64::STRDroX:
1046 case AArch64::STRHHroW:
1047 case AArch64::STRHHroX:
1048 case AArch64::STRHroW:
1049 case AArch64::STRHroX:
1050 case AArch64::STRQroW:
1051 case AArch64::STRQroX:
1052 case AArch64::STRSroW:
1053 case AArch64::STRSroX:
1054 case AArch64::STRWroW:
1055 case AArch64::STRWroX:
1056 case AArch64::STRXroW:
1057 case AArch64::STRXroX: {
1058 unsigned IsSigned =
MI.getOperand(3).getImm();
1065 unsigned Opc =
MI.getOpcode();
1069 case AArch64::SEH_StackAlloc:
1070 case AArch64::SEH_SaveFPLR:
1071 case AArch64::SEH_SaveFPLR_X:
1072 case AArch64::SEH_SaveReg:
1073 case AArch64::SEH_SaveReg_X:
1074 case AArch64::SEH_SaveRegP:
1075 case AArch64::SEH_SaveRegP_X:
1076 case AArch64::SEH_SaveFReg:
1077 case AArch64::SEH_SaveFReg_X:
1078 case AArch64::SEH_SaveFRegP:
1079 case AArch64::SEH_SaveFRegP_X:
1080 case AArch64::SEH_SetFP:
1081 case AArch64::SEH_AddFP:
1082 case AArch64::SEH_Nop:
1083 case AArch64::SEH_PrologEnd:
1084 case AArch64::SEH_EpilogStart:
1085 case AArch64::SEH_EpilogEnd:
1086 case AArch64::SEH_PACSignLR:
1087 case AArch64::SEH_SaveAnyRegQP:
1088 case AArch64::SEH_SaveAnyRegQPX:
1095 unsigned &SubIdx)
const {
1096 switch (
MI.getOpcode()) {
1099 case AArch64::SBFMXri:
1100 case AArch64::UBFMXri:
1103 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1106 SrcReg =
MI.getOperand(1).getReg();
1107 DstReg =
MI.getOperand(0).getReg();
1108 SubIdx = AArch64::sub_32;
1117 int64_t OffsetA = 0, OffsetB = 0;
1118 TypeSize WidthA(0,
false), WidthB(0,
false);
1119 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1140 OffsetAIsScalable == OffsetBIsScalable) {
1141 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1142 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1143 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1144 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1162 switch (
MI.getOpcode()) {
1165 if (
MI.getOperand(0).getImm() == 0x14)
1172 case AArch64::MSRpstatesvcrImm1:
1179 auto Next = std::next(
MI.getIterator());
1180 return Next !=
MBB->
end() && Next->isCFIInstruction();
1187 Register &SrcReg2, int64_t &CmpMask,
1188 int64_t &CmpValue)
const {
1191 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1192 if (!
MI.getOperand(1).isReg())
1195 switch (
MI.getOpcode()) {
1198 case AArch64::PTEST_PP:
1199 case AArch64::PTEST_PP_ANY:
1200 SrcReg =
MI.getOperand(0).getReg();
1201 SrcReg2 =
MI.getOperand(1).getReg();
1206 case AArch64::SUBSWrr:
1207 case AArch64::SUBSWrs:
1208 case AArch64::SUBSWrx:
1209 case AArch64::SUBSXrr:
1210 case AArch64::SUBSXrs:
1211 case AArch64::SUBSXrx:
1212 case AArch64::ADDSWrr:
1213 case AArch64::ADDSWrs:
1214 case AArch64::ADDSWrx:
1215 case AArch64::ADDSXrr:
1216 case AArch64::ADDSXrs:
1217 case AArch64::ADDSXrx:
1219 SrcReg =
MI.getOperand(1).getReg();
1220 SrcReg2 =
MI.getOperand(2).getReg();
1224 case AArch64::SUBSWri:
1225 case AArch64::ADDSWri:
1226 case AArch64::SUBSXri:
1227 case AArch64::ADDSXri:
1228 SrcReg =
MI.getOperand(1).getReg();
1231 CmpValue =
MI.getOperand(2).getImm();
1233 case AArch64::ANDSWri:
1234 case AArch64::ANDSXri:
1237 SrcReg =
MI.getOperand(1).getReg();
1241 MI.getOperand(2).getImm(),
1242 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1251 assert(
MBB &&
"Can't get MachineBasicBlock here");
1253 assert(MF &&
"Can't get MachineFunction here");
1258 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1262 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1265 if (!OpRegCstraints)
1273 "Operand has register constraints without being a register!");
1276 if (Reg.isPhysical()) {
1277 if (!OpRegCstraints->
contains(Reg))
1280 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1293 bool MIDefinesZeroReg =
false;
1294 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1295 MI.definesRegister(AArch64::XZR,
nullptr))
1296 MIDefinesZeroReg =
true;
1298 switch (
MI.getOpcode()) {
1300 return MI.getOpcode();
1301 case AArch64::ADDSWrr:
1302 return AArch64::ADDWrr;
1303 case AArch64::ADDSWri:
1304 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1305 case AArch64::ADDSWrs:
1306 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1307 case AArch64::ADDSWrx:
1308 return AArch64::ADDWrx;
1309 case AArch64::ADDSXrr:
1310 return AArch64::ADDXrr;
1311 case AArch64::ADDSXri:
1312 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1313 case AArch64::ADDSXrs:
1314 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1315 case AArch64::ADDSXrx:
1316 return AArch64::ADDXrx;
1317 case AArch64::SUBSWrr:
1318 return AArch64::SUBWrr;
1319 case AArch64::SUBSWri:
1320 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1321 case AArch64::SUBSWrs:
1322 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1323 case AArch64::SUBSWrx:
1324 return AArch64::SUBWrx;
1325 case AArch64::SUBSXrr:
1326 return AArch64::SUBXrr;
1327 case AArch64::SUBSXri:
1328 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1329 case AArch64::SUBSXrs:
1330 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1331 case AArch64::SUBSXrx:
1332 return AArch64::SUBXrx;
1347 if (To == To->getParent()->begin())
1352 if (To->getParent() !=
From->getParent())
1364 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1365 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1371std::optional<unsigned>
1375 unsigned MaskOpcode =
Mask->getOpcode();
1376 unsigned PredOpcode = Pred->
getOpcode();
1380 if (PredIsWhileLike) {
1384 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1398 if (PredIsPTestLike) {
1403 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1413 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1442 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1450 switch (PredOpcode) {
1451 case AArch64::AND_PPzPP:
1452 case AArch64::BIC_PPzPP:
1453 case AArch64::EOR_PPzPP:
1454 case AArch64::NAND_PPzPP:
1455 case AArch64::NOR_PPzPP:
1456 case AArch64::ORN_PPzPP:
1457 case AArch64::ORR_PPzPP:
1458 case AArch64::BRKA_PPzP:
1459 case AArch64::BRKPA_PPzPP:
1460 case AArch64::BRKB_PPzP:
1461 case AArch64::BRKPB_PPzPP:
1462 case AArch64::RDFFR_PPz: {
1466 if (Mask != PredMask)
1470 case AArch64::BRKN_PPzP: {
1474 if ((MaskOpcode != AArch64::PTRUE_B) ||
1475 (
Mask->getOperand(1).getImm() != 31))
1479 case AArch64::PTRUE_B:
1492bool AArch64InstrInfo::optimizePTestInstr(
1493 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1495 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1496 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1497 unsigned PredOpcode = Pred->
getOpcode();
1498 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1514 if (*NewOp != PredOpcode) {
1525 for (; i !=
e; ++i) {
1556 if (DeadNZCVIdx != -1) {
1575 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1576 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1577 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1586 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1588 return (CmpValue == 0 || CmpValue == 1) &&
1589 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1597 switch (Instr.getOpcode()) {
1599 return AArch64::INSTRUCTION_LIST_END;
1601 case AArch64::ADDSWrr:
1602 case AArch64::ADDSWri:
1603 case AArch64::ADDSXrr:
1604 case AArch64::ADDSXri:
1605 case AArch64::SUBSWrr:
1606 case AArch64::SUBSWri:
1607 case AArch64::SUBSXrr:
1608 case AArch64::SUBSXri:
1609 return Instr.getOpcode();
1611 case AArch64::ADDWrr:
1612 return AArch64::ADDSWrr;
1613 case AArch64::ADDWri:
1614 return AArch64::ADDSWri;
1615 case AArch64::ADDXrr:
1616 return AArch64::ADDSXrr;
1617 case AArch64::ADDXri:
1618 return AArch64::ADDSXri;
1619 case AArch64::ADCWr:
1620 return AArch64::ADCSWr;
1621 case AArch64::ADCXr:
1622 return AArch64::ADCSXr;
1623 case AArch64::SUBWrr:
1624 return AArch64::SUBSWrr;
1625 case AArch64::SUBWri:
1626 return AArch64::SUBSWri;
1627 case AArch64::SUBXrr:
1628 return AArch64::SUBSXrr;
1629 case AArch64::SUBXri:
1630 return AArch64::SUBSXri;
1631 case AArch64::SBCWr:
1632 return AArch64::SBCSWr;
1633 case AArch64::SBCXr:
1634 return AArch64::SBCSXr;
1635 case AArch64::ANDWri:
1636 return AArch64::ANDSWri;
1637 case AArch64::ANDXri:
1638 return AArch64::ANDSXri;
1645 if (BB->isLiveIn(AArch64::NZCV))
1654 switch (Instr.getOpcode()) {
1658 case AArch64::Bcc: {
1659 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1664 case AArch64::CSINVWr:
1665 case AArch64::CSINVXr:
1666 case AArch64::CSINCWr:
1667 case AArch64::CSINCXr:
1668 case AArch64::CSELWr:
1669 case AArch64::CSELXr:
1670 case AArch64::CSNEGWr:
1671 case AArch64::CSNEGXr:
1672 case AArch64::FCSELSrrr:
1673 case AArch64::FCSELDrrr: {
1674 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1687 Instr.getOperand(CCIdx).getImm())
1740std::optional<UsedNZCV>
1745 if (
MI.getParent() != CmpParent)
1746 return std::nullopt;
1749 return std::nullopt;
1754 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1757 return std::nullopt;
1762 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1765 return NZCVUsedAfterCmp;
1769 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1773 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1795 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1801 "Caller guarantees that CmpInstr compares with constant 0");
1804 if (!NZVCUsed || NZVCUsed->C)
1826bool AArch64InstrInfo::substituteCmpToZero(
1837 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1844 MI->setDesc(
get(NewOpc));
1849 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1861 assert((CmpValue == 0 || CmpValue == 1) &&
1862 "Only comparisons to 0 or 1 considered for removal!");
1865 unsigned MIOpc =
MI.getOpcode();
1866 if (MIOpc == AArch64::CSINCWr) {
1867 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1868 MI.getOperand(2).getReg() != AArch64::WZR)
1870 }
else if (MIOpc == AArch64::CSINCXr) {
1871 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1872 MI.getOperand(2).getReg() != AArch64::XZR)
1882 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1886 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1888 if (CmpValue && !IsSubsRegImm)
1890 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1895 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1898 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1902 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1905 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1906 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1909 if (MIUsedNZCV.
N && !CmpValue)
1951bool AArch64InstrInfo::removeCmpToZeroOrOne(
1959 bool IsInvertCC =
false;
1969 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1980 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1981 MI.getOpcode() != AArch64::CATCHRET)
1989 if (
MI.getOpcode() == AArch64::CATCHRET) {
1998 FirstEpilogSEH = std::prev(FirstEpilogSEH);
2000 FirstEpilogSEH = std::next(FirstEpilogSEH);
2015 if (M.getStackProtectorGuard() ==
"sysreg") {
2017 AArch64SysReg::lookupSysRegByName(M.getStackProtectorGuardReg());
2025 int Offset = M.getStackProtectorGuardOffset();
2077 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2086 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2128 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2151 switch (
MI.getOpcode()) {
2154 case AArch64::MOVZWi:
2155 case AArch64::MOVZXi:
2156 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2157 assert(
MI.getDesc().getNumOperands() == 3 &&
2158 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2162 case AArch64::ANDWri:
2163 return MI.getOperand(1).getReg() == AArch64::WZR;
2164 case AArch64::ANDXri:
2165 return MI.getOperand(1).getReg() == AArch64::XZR;
2166 case TargetOpcode::COPY:
2167 return MI.getOperand(1).getReg() == AArch64::WZR;
2175 switch (
MI.getOpcode()) {
2178 case TargetOpcode::COPY: {
2181 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2182 AArch64::GPR64RegClass.
contains(DstReg));
2184 case AArch64::ORRXrs:
2185 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2186 assert(
MI.getDesc().getNumOperands() == 4 &&
2187 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2191 case AArch64::ADDXri:
2192 if (
MI.getOperand(2).getImm() == 0) {
2193 assert(
MI.getDesc().getNumOperands() == 4 &&
2194 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2205 switch (
MI.getOpcode()) {
2208 case TargetOpcode::COPY: {
2210 return AArch64::FPR128RegClass.contains(DstReg);
2212 case AArch64::ORRv16i8:
2213 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2214 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2215 "invalid ORRv16i8 operands");
2224 int &FrameIndex)
const {
2225 switch (
MI.getOpcode()) {
2228 case AArch64::LDRWui:
2229 case AArch64::LDRXui:
2230 case AArch64::LDRBui:
2231 case AArch64::LDRHui:
2232 case AArch64::LDRSui:
2233 case AArch64::LDRDui:
2234 case AArch64::LDRQui:
2235 case AArch64::LDR_PXI:
2236 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2237 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2238 FrameIndex =
MI.getOperand(1).getIndex();
2239 return MI.getOperand(0).getReg();
2248 int &FrameIndex)
const {
2249 switch (
MI.getOpcode()) {
2252 case AArch64::STRWui:
2253 case AArch64::STRXui:
2254 case AArch64::STRBui:
2255 case AArch64::STRHui:
2256 case AArch64::STRSui:
2257 case AArch64::STRDui:
2258 case AArch64::STRQui:
2259 case AArch64::STR_PXI:
2260 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2261 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2262 FrameIndex =
MI.getOperand(1).getIndex();
2263 return MI.getOperand(0).getReg();
2273 return MMO->getFlags() & MOSuppressPair;
2279 if (
MI.memoperands_empty())
2287 return MMO->getFlags() & MOStridedAccess;
2295 case AArch64::STURSi:
2296 case AArch64::STRSpre:
2297 case AArch64::STURDi:
2298 case AArch64::STRDpre:
2299 case AArch64::STURQi:
2300 case AArch64::STRQpre:
2301 case AArch64::STURBBi:
2302 case AArch64::STURHHi:
2303 case AArch64::STURWi:
2304 case AArch64::STRWpre:
2305 case AArch64::STURXi:
2306 case AArch64::STRXpre:
2307 case AArch64::LDURSi:
2308 case AArch64::LDRSpre:
2309 case AArch64::LDURDi:
2310 case AArch64::LDRDpre:
2311 case AArch64::LDURQi:
2312 case AArch64::LDRQpre:
2313 case AArch64::LDURWi:
2314 case AArch64::LDRWpre:
2315 case AArch64::LDURXi:
2316 case AArch64::LDRXpre:
2317 case AArch64::LDRSWpre:
2318 case AArch64::LDURSWi:
2319 case AArch64::LDURHHi:
2320 case AArch64::LDURBBi:
2321 case AArch64::LDURSBWi:
2322 case AArch64::LDURSHWi:
2330 case AArch64::PRFMui:
return AArch64::PRFUMi;
2331 case AArch64::LDRXui:
return AArch64::LDURXi;
2332 case AArch64::LDRWui:
return AArch64::LDURWi;
2333 case AArch64::LDRBui:
return AArch64::LDURBi;
2334 case AArch64::LDRHui:
return AArch64::LDURHi;
2335 case AArch64::LDRSui:
return AArch64::LDURSi;
2336 case AArch64::LDRDui:
return AArch64::LDURDi;
2337 case AArch64::LDRQui:
return AArch64::LDURQi;
2338 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2339 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2340 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2341 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2342 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2343 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2344 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2345 case AArch64::STRXui:
return AArch64::STURXi;
2346 case AArch64::STRWui:
return AArch64::STURWi;
2347 case AArch64::STRBui:
return AArch64::STURBi;
2348 case AArch64::STRHui:
return AArch64::STURHi;
2349 case AArch64::STRSui:
return AArch64::STURSi;
2350 case AArch64::STRDui:
return AArch64::STURDi;
2351 case AArch64::STRQui:
return AArch64::STURQi;
2352 case AArch64::STRBBui:
return AArch64::STURBBi;
2353 case AArch64::STRHHui:
return AArch64::STURHHi;
2362 case AArch64::LDAPURBi:
2363 case AArch64::LDAPURHi:
2364 case AArch64::LDAPURi:
2365 case AArch64::LDAPURSBWi:
2366 case AArch64::LDAPURSBXi:
2367 case AArch64::LDAPURSHWi:
2368 case AArch64::LDAPURSHXi:
2369 case AArch64::LDAPURSWi:
2370 case AArch64::LDAPURXi:
2371 case AArch64::LDR_PPXI:
2372 case AArch64::LDR_PXI:
2373 case AArch64::LDR_ZXI:
2374 case AArch64::LDR_ZZXI:
2375 case AArch64::LDR_ZZZXI:
2376 case AArch64::LDR_ZZZZXI:
2377 case AArch64::LDRBBui:
2378 case AArch64::LDRBui:
2379 case AArch64::LDRDui:
2380 case AArch64::LDRHHui:
2381 case AArch64::LDRHui:
2382 case AArch64::LDRQui:
2383 case AArch64::LDRSBWui:
2384 case AArch64::LDRSBXui:
2385 case AArch64::LDRSHWui:
2386 case AArch64::LDRSHXui:
2387 case AArch64::LDRSui:
2388 case AArch64::LDRSWui:
2389 case AArch64::LDRWui:
2390 case AArch64::LDRXui:
2391 case AArch64::LDURBBi:
2392 case AArch64::LDURBi:
2393 case AArch64::LDURDi:
2394 case AArch64::LDURHHi:
2395 case AArch64::LDURHi:
2396 case AArch64::LDURQi:
2397 case AArch64::LDURSBWi:
2398 case AArch64::LDURSBXi:
2399 case AArch64::LDURSHWi:
2400 case AArch64::LDURSHXi:
2401 case AArch64::LDURSi:
2402 case AArch64::LDURSWi:
2403 case AArch64::LDURWi:
2404 case AArch64::LDURXi:
2405 case AArch64::PRFMui:
2406 case AArch64::PRFUMi:
2407 case AArch64::ST2Gi:
2409 case AArch64::STLURBi:
2410 case AArch64::STLURHi:
2411 case AArch64::STLURWi:
2412 case AArch64::STLURXi:
2413 case AArch64::StoreSwiftAsyncContext:
2414 case AArch64::STR_PPXI:
2415 case AArch64::STR_PXI:
2416 case AArch64::STR_ZXI:
2417 case AArch64::STR_ZZXI:
2418 case AArch64::STR_ZZZXI:
2419 case AArch64::STR_ZZZZXI:
2420 case AArch64::STRBBui:
2421 case AArch64::STRBui:
2422 case AArch64::STRDui:
2423 case AArch64::STRHHui:
2424 case AArch64::STRHui:
2425 case AArch64::STRQui:
2426 case AArch64::STRSui:
2427 case AArch64::STRWui:
2428 case AArch64::STRXui:
2429 case AArch64::STURBBi:
2430 case AArch64::STURBi:
2431 case AArch64::STURDi:
2432 case AArch64::STURHHi:
2433 case AArch64::STURHi:
2434 case AArch64::STURQi:
2435 case AArch64::STURSi:
2436 case AArch64::STURWi:
2437 case AArch64::STURXi:
2438 case AArch64::STZ2Gi:
2439 case AArch64::STZGi:
2440 case AArch64::TAGPstack:
2441 case AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO:
2442 case AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO:
2444 case AArch64::LD1B_D_IMM:
2445 case AArch64::LD1B_H_IMM:
2446 case AArch64::LD1B_IMM:
2447 case AArch64::LD1B_S_IMM:
2448 case AArch64::LD1D_IMM:
2449 case AArch64::LD1H_D_IMM:
2450 case AArch64::LD1H_IMM:
2451 case AArch64::LD1H_S_IMM:
2452 case AArch64::LD1RB_D_IMM:
2453 case AArch64::LD1RB_H_IMM:
2454 case AArch64::LD1RB_IMM:
2455 case AArch64::LD1RB_S_IMM:
2456 case AArch64::LD1RD_IMM:
2457 case AArch64::LD1RH_D_IMM:
2458 case AArch64::LD1RH_IMM:
2459 case AArch64::LD1RH_S_IMM:
2460 case AArch64::LD1RSB_D_IMM:
2461 case AArch64::LD1RSB_H_IMM:
2462 case AArch64::LD1RSB_S_IMM:
2463 case AArch64::LD1RSH_D_IMM:
2464 case AArch64::LD1RSH_S_IMM:
2465 case AArch64::LD1RSW_IMM:
2466 case AArch64::LD1RW_D_IMM:
2467 case AArch64::LD1RW_IMM:
2468 case AArch64::LD1SB_D_IMM:
2469 case AArch64::LD1SB_H_IMM:
2470 case AArch64::LD1SB_S_IMM:
2471 case AArch64::LD1SH_D_IMM:
2472 case AArch64::LD1SH_S_IMM:
2473 case AArch64::LD1SW_D_IMM:
2474 case AArch64::LD1W_D_IMM:
2475 case AArch64::LD1W_IMM:
2476 case AArch64::LD2B_IMM:
2477 case AArch64::LD2D_IMM:
2478 case AArch64::LD2H_IMM:
2479 case AArch64::LD2W_IMM:
2480 case AArch64::LD3B_IMM:
2481 case AArch64::LD3D_IMM:
2482 case AArch64::LD3H_IMM:
2483 case AArch64::LD3W_IMM:
2484 case AArch64::LD4B_IMM:
2485 case AArch64::LD4D_IMM:
2486 case AArch64::LD4H_IMM:
2487 case AArch64::LD4W_IMM:
2489 case AArch64::LDNF1B_D_IMM:
2490 case AArch64::LDNF1B_H_IMM:
2491 case AArch64::LDNF1B_IMM:
2492 case AArch64::LDNF1B_S_IMM:
2493 case AArch64::LDNF1D_IMM:
2494 case AArch64::LDNF1H_D_IMM:
2495 case AArch64::LDNF1H_IMM:
2496 case AArch64::LDNF1H_S_IMM:
2497 case AArch64::LDNF1SB_D_IMM:
2498 case AArch64::LDNF1SB_H_IMM:
2499 case AArch64::LDNF1SB_S_IMM:
2500 case AArch64::LDNF1SH_D_IMM:
2501 case AArch64::LDNF1SH_S_IMM:
2502 case AArch64::LDNF1SW_D_IMM:
2503 case AArch64::LDNF1W_D_IMM:
2504 case AArch64::LDNF1W_IMM:
2505 case AArch64::LDNPDi:
2506 case AArch64::LDNPQi:
2507 case AArch64::LDNPSi:
2508 case AArch64::LDNPWi:
2509 case AArch64::LDNPXi:
2510 case AArch64::LDNT1B_ZRI:
2511 case AArch64::LDNT1D_ZRI:
2512 case AArch64::LDNT1H_ZRI:
2513 case AArch64::LDNT1W_ZRI:
2514 case AArch64::LDPDi:
2515 case AArch64::LDPQi:
2516 case AArch64::LDPSi:
2517 case AArch64::LDPWi:
2518 case AArch64::LDPXi:
2519 case AArch64::LDRBBpost:
2520 case AArch64::LDRBBpre:
2521 case AArch64::LDRBpost:
2522 case AArch64::LDRBpre:
2523 case AArch64::LDRDpost:
2524 case AArch64::LDRDpre:
2525 case AArch64::LDRHHpost:
2526 case AArch64::LDRHHpre:
2527 case AArch64::LDRHpost:
2528 case AArch64::LDRHpre:
2529 case AArch64::LDRQpost:
2530 case AArch64::LDRQpre:
2531 case AArch64::LDRSpost:
2532 case AArch64::LDRSpre:
2533 case AArch64::LDRWpost:
2534 case AArch64::LDRWpre:
2535 case AArch64::LDRXpost:
2536 case AArch64::LDRXpre:
2537 case AArch64::ST1B_D_IMM:
2538 case AArch64::ST1B_H_IMM:
2539 case AArch64::ST1B_IMM:
2540 case AArch64::ST1B_S_IMM:
2541 case AArch64::ST1D_IMM:
2542 case AArch64::ST1H_D_IMM:
2543 case AArch64::ST1H_IMM:
2544 case AArch64::ST1H_S_IMM:
2545 case AArch64::ST1W_D_IMM:
2546 case AArch64::ST1W_IMM:
2547 case AArch64::ST2B_IMM:
2548 case AArch64::ST2D_IMM:
2549 case AArch64::ST2H_IMM:
2550 case AArch64::ST2W_IMM:
2551 case AArch64::ST3B_IMM:
2552 case AArch64::ST3D_IMM:
2553 case AArch64::ST3H_IMM:
2554 case AArch64::ST3W_IMM:
2555 case AArch64::ST4B_IMM:
2556 case AArch64::ST4D_IMM:
2557 case AArch64::ST4H_IMM:
2558 case AArch64::ST4W_IMM:
2559 case AArch64::STGPi:
2560 case AArch64::STGPreIndex:
2561 case AArch64::STZGPreIndex:
2562 case AArch64::ST2GPreIndex:
2563 case AArch64::STZ2GPreIndex:
2564 case AArch64::STGPostIndex:
2565 case AArch64::STZGPostIndex:
2566 case AArch64::ST2GPostIndex:
2567 case AArch64::STZ2GPostIndex:
2568 case AArch64::STNPDi:
2569 case AArch64::STNPQi:
2570 case AArch64::STNPSi:
2571 case AArch64::STNPWi:
2572 case AArch64::STNPXi:
2573 case AArch64::STNT1B_ZRI:
2574 case AArch64::STNT1D_ZRI:
2575 case AArch64::STNT1H_ZRI:
2576 case AArch64::STNT1W_ZRI:
2577 case AArch64::STPDi:
2578 case AArch64::STPQi:
2579 case AArch64::STPSi:
2580 case AArch64::STPWi:
2581 case AArch64::STPXi:
2582 case AArch64::STRBBpost:
2583 case AArch64::STRBBpre:
2584 case AArch64::STRBpost:
2585 case AArch64::STRBpre:
2586 case AArch64::STRDpost:
2587 case AArch64::STRDpre:
2588 case AArch64::STRHHpost:
2589 case AArch64::STRHHpre:
2590 case AArch64::STRHpost:
2591 case AArch64::STRHpre:
2592 case AArch64::STRQpost:
2593 case AArch64::STRQpre:
2594 case AArch64::STRSpost:
2595 case AArch64::STRSpre:
2596 case AArch64::STRWpost:
2597 case AArch64::STRWpre:
2598 case AArch64::STRXpost:
2599 case AArch64::STRXpre:
2601 case AArch64::LDPDpost:
2602 case AArch64::LDPDpre:
2603 case AArch64::LDPQpost:
2604 case AArch64::LDPQpre:
2605 case AArch64::LDPSpost:
2606 case AArch64::LDPSpre:
2607 case AArch64::LDPWpost:
2608 case AArch64::LDPWpre:
2609 case AArch64::LDPXpost:
2610 case AArch64::LDPXpre:
2611 case AArch64::STGPpre:
2612 case AArch64::STGPpost:
2613 case AArch64::STPDpost:
2614 case AArch64::STPDpre:
2615 case AArch64::STPQpost:
2616 case AArch64::STPQpre:
2617 case AArch64::STPSpost:
2618 case AArch64::STPSpre:
2619 case AArch64::STPWpost:
2620 case AArch64::STPWpre:
2621 case AArch64::STPXpost:
2622 case AArch64::STPXpre:
2628 switch (
MI.getOpcode()) {
2632 case AArch64::STRSui:
2633 case AArch64::STRDui:
2634 case AArch64::STRQui:
2635 case AArch64::STRXui:
2636 case AArch64::STRWui:
2637 case AArch64::LDRSui:
2638 case AArch64::LDRDui:
2639 case AArch64::LDRQui:
2640 case AArch64::LDRXui:
2641 case AArch64::LDRWui:
2642 case AArch64::LDRSWui:
2644 case AArch64::STURSi:
2645 case AArch64::STRSpre:
2646 case AArch64::STURDi:
2647 case AArch64::STRDpre:
2648 case AArch64::STURQi:
2649 case AArch64::STRQpre:
2650 case AArch64::STURWi:
2651 case AArch64::STRWpre:
2652 case AArch64::STURXi:
2653 case AArch64::STRXpre:
2654 case AArch64::LDURSi:
2655 case AArch64::LDRSpre:
2656 case AArch64::LDURDi:
2657 case AArch64::LDRDpre:
2658 case AArch64::LDURQi:
2659 case AArch64::LDRQpre:
2660 case AArch64::LDURWi:
2661 case AArch64::LDRWpre:
2662 case AArch64::LDURXi:
2663 case AArch64::LDRXpre:
2664 case AArch64::LDURSWi:
2665 case AArch64::LDRSWpre:
2671 switch (
MI.getOpcode()) {
2674 "Unexpected instruction - was a new tail call opcode introduced?");
2676 case AArch64::TCRETURNdi:
2677 case AArch64::TCRETURNri:
2678 case AArch64::TCRETURNrix16x17:
2679 case AArch64::TCRETURNrix17:
2680 case AArch64::TCRETURNrinotx16:
2681 case AArch64::TCRETURNriALL:
2682 case AArch64::AUTH_TCRETURN:
2683 case AArch64::AUTH_TCRETURN_BTI:
2693 case AArch64::ADDWri:
2694 return AArch64::ADDSWri;
2695 case AArch64::ADDWrr:
2696 return AArch64::ADDSWrr;
2697 case AArch64::ADDWrs:
2698 return AArch64::ADDSWrs;
2699 case AArch64::ADDWrx:
2700 return AArch64::ADDSWrx;
2701 case AArch64::ANDWri:
2702 return AArch64::ANDSWri;
2703 case AArch64::ANDWrr:
2704 return AArch64::ANDSWrr;
2705 case AArch64::ANDWrs:
2706 return AArch64::ANDSWrs;
2707 case AArch64::BICWrr:
2708 return AArch64::BICSWrr;
2709 case AArch64::BICWrs:
2710 return AArch64::BICSWrs;
2711 case AArch64::SUBWri:
2712 return AArch64::SUBSWri;
2713 case AArch64::SUBWrr:
2714 return AArch64::SUBSWrr;
2715 case AArch64::SUBWrs:
2716 return AArch64::SUBSWrs;
2717 case AArch64::SUBWrx:
2718 return AArch64::SUBSWrx;
2720 case AArch64::ADDXri:
2721 return AArch64::ADDSXri;
2722 case AArch64::ADDXrr:
2723 return AArch64::ADDSXrr;
2724 case AArch64::ADDXrs:
2725 return AArch64::ADDSXrs;
2726 case AArch64::ADDXrx:
2727 return AArch64::ADDSXrx;
2728 case AArch64::ANDXri:
2729 return AArch64::ANDSXri;
2730 case AArch64::ANDXrr:
2731 return AArch64::ANDSXrr;
2732 case AArch64::ANDXrs:
2733 return AArch64::ANDSXrs;
2734 case AArch64::BICXrr:
2735 return AArch64::BICSXrr;
2736 case AArch64::BICXrs:
2737 return AArch64::BICSXrs;
2738 case AArch64::SUBXri:
2739 return AArch64::SUBSXri;
2740 case AArch64::SUBXrr:
2741 return AArch64::SUBSXrr;
2742 case AArch64::SUBXrs:
2743 return AArch64::SUBSXrs;
2744 case AArch64::SUBXrx:
2745 return AArch64::SUBSXrx;
2747 case AArch64::AND_PPzPP:
2748 return AArch64::ANDS_PPzPP;
2749 case AArch64::BIC_PPzPP:
2750 return AArch64::BICS_PPzPP;
2751 case AArch64::EOR_PPzPP:
2752 return AArch64::EORS_PPzPP;
2753 case AArch64::NAND_PPzPP:
2754 return AArch64::NANDS_PPzPP;
2755 case AArch64::NOR_PPzPP:
2756 return AArch64::NORS_PPzPP;
2757 case AArch64::ORN_PPzPP:
2758 return AArch64::ORNS_PPzPP;
2759 case AArch64::ORR_PPzPP:
2760 return AArch64::ORRS_PPzPP;
2761 case AArch64::BRKA_PPzP:
2762 return AArch64::BRKAS_PPzP;
2763 case AArch64::BRKPA_PPzPP:
2764 return AArch64::BRKPAS_PPzPP;
2765 case AArch64::BRKB_PPzP:
2766 return AArch64::BRKBS_PPzP;
2767 case AArch64::BRKPB_PPzPP:
2768 return AArch64::BRKPBS_PPzPP;
2769 case AArch64::BRKN_PPzP:
2770 return AArch64::BRKNS_PPzP;
2771 case AArch64::RDFFR_PPz:
2772 return AArch64::RDFFRS_PPz;
2773 case AArch64::PTRUE_B:
2774 return AArch64::PTRUES_B;
2785 if (
MI.hasOrderedMemoryRef())
2790 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2791 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2792 "Expected a reg or frame index operand.");
2796 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2798 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2811 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2812 Register BaseReg =
MI.getOperand(1).getReg();
2814 if (
MI.modifiesRegister(BaseReg,
TRI))
2827 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2829 MI.getMF()->getFunction().needsUnwindTableEntry();
2835 if (Subtarget.isPaired128Slow()) {
2836 switch (
MI.getOpcode()) {
2839 case AArch64::LDURQi:
2840 case AArch64::STURQi:
2841 case AArch64::LDRQui:
2842 case AArch64::STRQui:
2869std::optional<ExtAddrMode>
2874 bool OffsetIsScalable;
2875 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2876 return std::nullopt;
2879 return std::nullopt;
2894 int64_t OffsetScale = 1;
2899 case AArch64::LDURQi:
2900 case AArch64::STURQi:
2904 case AArch64::LDURDi:
2905 case AArch64::STURDi:
2906 case AArch64::LDURXi:
2907 case AArch64::STURXi:
2911 case AArch64::LDURWi:
2912 case AArch64::LDURSWi:
2913 case AArch64::STURWi:
2917 case AArch64::LDURHi:
2918 case AArch64::STURHi:
2919 case AArch64::LDURHHi:
2920 case AArch64::STURHHi:
2921 case AArch64::LDURSHXi:
2922 case AArch64::LDURSHWi:
2926 case AArch64::LDRBroX:
2927 case AArch64::LDRBBroX:
2928 case AArch64::LDRSBXroX:
2929 case AArch64::LDRSBWroX:
2930 case AArch64::STRBroX:
2931 case AArch64::STRBBroX:
2932 case AArch64::LDURBi:
2933 case AArch64::LDURBBi:
2934 case AArch64::LDURSBXi:
2935 case AArch64::LDURSBWi:
2936 case AArch64::STURBi:
2937 case AArch64::STURBBi:
2938 case AArch64::LDRBui:
2939 case AArch64::LDRBBui:
2940 case AArch64::LDRSBXui:
2941 case AArch64::LDRSBWui:
2942 case AArch64::STRBui:
2943 case AArch64::STRBBui:
2947 case AArch64::LDRQroX:
2948 case AArch64::STRQroX:
2949 case AArch64::LDRQui:
2950 case AArch64::STRQui:
2955 case AArch64::LDRDroX:
2956 case AArch64::STRDroX:
2957 case AArch64::LDRXroX:
2958 case AArch64::STRXroX:
2959 case AArch64::LDRDui:
2960 case AArch64::STRDui:
2961 case AArch64::LDRXui:
2962 case AArch64::STRXui:
2967 case AArch64::LDRWroX:
2968 case AArch64::LDRSWroX:
2969 case AArch64::STRWroX:
2970 case AArch64::LDRWui:
2971 case AArch64::LDRSWui:
2972 case AArch64::STRWui:
2977 case AArch64::LDRHroX:
2978 case AArch64::STRHroX:
2979 case AArch64::LDRHHroX:
2980 case AArch64::STRHHroX:
2981 case AArch64::LDRSHXroX:
2982 case AArch64::LDRSHWroX:
2983 case AArch64::LDRHui:
2984 case AArch64::STRHui:
2985 case AArch64::LDRHHui:
2986 case AArch64::STRHHui:
2987 case AArch64::LDRSHXui:
2988 case AArch64::LDRSHWui:
2996 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
3020 case AArch64::SBFMXri:
3033 AM.
Scale = OffsetScale;
3038 case TargetOpcode::SUBREG_TO_REG: {
3051 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
3055 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
3057 DefMI.getOperand(3).getImm() != 0)
3064 AM.
Scale = OffsetScale;
3075 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
3076 int64_t NewOffset) ->
bool {
3077 int64_t MinOffset, MaxOffset;
3094 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3095 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3097 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
3099 int64_t NewOffset = OldOffset + Disp;
3104 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3114 auto canFoldAddRegIntoAddrMode =
3131 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3132 Subtarget.isSTRQroSlow();
3141 case AArch64::ADDXri:
3147 return canFoldAddSubImmIntoAddrMode(Disp);
3149 case AArch64::SUBXri:
3155 return canFoldAddSubImmIntoAddrMode(-Disp);
3157 case AArch64::ADDXrs: {
3170 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3172 if (avoidSlowSTRQ(MemI))
3175 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3178 case AArch64::ADDXrr:
3186 if (!OptSize && avoidSlowSTRQ(MemI))
3188 return canFoldAddRegIntoAddrMode(1);
3190 case AArch64::ADDXrx:
3198 if (!OptSize && avoidSlowSTRQ(MemI))
3207 return canFoldAddRegIntoAddrMode(
3222 case AArch64::LDURQi:
3223 case AArch64::LDRQui:
3224 return AArch64::LDRQroX;
3225 case AArch64::STURQi:
3226 case AArch64::STRQui:
3227 return AArch64::STRQroX;
3228 case AArch64::LDURDi:
3229 case AArch64::LDRDui:
3230 return AArch64::LDRDroX;
3231 case AArch64::STURDi:
3232 case AArch64::STRDui:
3233 return AArch64::STRDroX;
3234 case AArch64::LDURXi:
3235 case AArch64::LDRXui:
3236 return AArch64::LDRXroX;
3237 case AArch64::STURXi:
3238 case AArch64::STRXui:
3239 return AArch64::STRXroX;
3240 case AArch64::LDURWi:
3241 case AArch64::LDRWui:
3242 return AArch64::LDRWroX;
3243 case AArch64::LDURSWi:
3244 case AArch64::LDRSWui:
3245 return AArch64::LDRSWroX;
3246 case AArch64::STURWi:
3247 case AArch64::STRWui:
3248 return AArch64::STRWroX;
3249 case AArch64::LDURHi:
3250 case AArch64::LDRHui:
3251 return AArch64::LDRHroX;
3252 case AArch64::STURHi:
3253 case AArch64::STRHui:
3254 return AArch64::STRHroX;
3255 case AArch64::LDURHHi:
3256 case AArch64::LDRHHui:
3257 return AArch64::LDRHHroX;
3258 case AArch64::STURHHi:
3259 case AArch64::STRHHui:
3260 return AArch64::STRHHroX;
3261 case AArch64::LDURSHXi:
3262 case AArch64::LDRSHXui:
3263 return AArch64::LDRSHXroX;
3264 case AArch64::LDURSHWi:
3265 case AArch64::LDRSHWui:
3266 return AArch64::LDRSHWroX;
3267 case AArch64::LDURBi:
3268 case AArch64::LDRBui:
3269 return AArch64::LDRBroX;
3270 case AArch64::LDURBBi:
3271 case AArch64::LDRBBui:
3272 return AArch64::LDRBBroX;
3273 case AArch64::LDURSBXi:
3274 case AArch64::LDRSBXui:
3275 return AArch64::LDRSBXroX;
3276 case AArch64::LDURSBWi:
3277 case AArch64::LDRSBWui:
3278 return AArch64::LDRSBWroX;
3279 case AArch64::STURBi:
3280 case AArch64::STRBui:
3281 return AArch64::STRBroX;
3282 case AArch64::STURBBi:
3283 case AArch64::STRBBui:
3284 return AArch64::STRBBroX;
3296 case AArch64::LDURQi:
3298 return AArch64::LDRQui;
3299 case AArch64::STURQi:
3301 return AArch64::STRQui;
3302 case AArch64::LDURDi:
3304 return AArch64::LDRDui;
3305 case AArch64::STURDi:
3307 return AArch64::STRDui;
3308 case AArch64::LDURXi:
3310 return AArch64::LDRXui;
3311 case AArch64::STURXi:
3313 return AArch64::STRXui;
3314 case AArch64::LDURWi:
3316 return AArch64::LDRWui;
3317 case AArch64::LDURSWi:
3319 return AArch64::LDRSWui;
3320 case AArch64::STURWi:
3322 return AArch64::STRWui;
3323 case AArch64::LDURHi:
3325 return AArch64::LDRHui;
3326 case AArch64::STURHi:
3328 return AArch64::STRHui;
3329 case AArch64::LDURHHi:
3331 return AArch64::LDRHHui;
3332 case AArch64::STURHHi:
3334 return AArch64::STRHHui;
3335 case AArch64::LDURSHXi:
3337 return AArch64::LDRSHXui;
3338 case AArch64::LDURSHWi:
3340 return AArch64::LDRSHWui;
3341 case AArch64::LDURBi:
3343 return AArch64::LDRBui;
3344 case AArch64::LDURBBi:
3346 return AArch64::LDRBBui;
3347 case AArch64::LDURSBXi:
3349 return AArch64::LDRSBXui;
3350 case AArch64::LDURSBWi:
3352 return AArch64::LDRSBWui;
3353 case AArch64::STURBi:
3355 return AArch64::STRBui;
3356 case AArch64::STURBBi:
3358 return AArch64::STRBBui;
3359 case AArch64::LDRQui:
3360 case AArch64::STRQui:
3363 case AArch64::LDRDui:
3364 case AArch64::STRDui:
3365 case AArch64::LDRXui:
3366 case AArch64::STRXui:
3369 case AArch64::LDRWui:
3370 case AArch64::LDRSWui:
3371 case AArch64::STRWui:
3374 case AArch64::LDRHui:
3375 case AArch64::STRHui:
3376 case AArch64::LDRHHui:
3377 case AArch64::STRHHui:
3378 case AArch64::LDRSHXui:
3379 case AArch64::LDRSHWui:
3382 case AArch64::LDRBui:
3383 case AArch64::LDRBBui:
3384 case AArch64::LDRSBXui:
3385 case AArch64::LDRSBWui:
3386 case AArch64::STRBui:
3387 case AArch64::STRBBui:
3401 case AArch64::LDURQi:
3402 case AArch64::STURQi:
3403 case AArch64::LDURDi:
3404 case AArch64::STURDi:
3405 case AArch64::LDURXi:
3406 case AArch64::STURXi:
3407 case AArch64::LDURWi:
3408 case AArch64::LDURSWi:
3409 case AArch64::STURWi:
3410 case AArch64::LDURHi:
3411 case AArch64::STURHi:
3412 case AArch64::LDURHHi:
3413 case AArch64::STURHHi:
3414 case AArch64::LDURSHXi:
3415 case AArch64::LDURSHWi:
3416 case AArch64::LDURBi:
3417 case AArch64::STURBi:
3418 case AArch64::LDURBBi:
3419 case AArch64::STURBBi:
3420 case AArch64::LDURSBWi:
3421 case AArch64::LDURSBXi:
3423 case AArch64::LDRQui:
3424 return AArch64::LDURQi;
3425 case AArch64::STRQui:
3426 return AArch64::STURQi;
3427 case AArch64::LDRDui:
3428 return AArch64::LDURDi;
3429 case AArch64::STRDui:
3430 return AArch64::STURDi;
3431 case AArch64::LDRXui:
3432 return AArch64::LDURXi;
3433 case AArch64::STRXui:
3434 return AArch64::STURXi;
3435 case AArch64::LDRWui:
3436 return AArch64::LDURWi;
3437 case AArch64::LDRSWui:
3438 return AArch64::LDURSWi;
3439 case AArch64::STRWui:
3440 return AArch64::STURWi;
3441 case AArch64::LDRHui:
3442 return AArch64::LDURHi;
3443 case AArch64::STRHui:
3444 return AArch64::STURHi;
3445 case AArch64::LDRHHui:
3446 return AArch64::LDURHHi;
3447 case AArch64::STRHHui:
3448 return AArch64::STURHHi;
3449 case AArch64::LDRSHXui:
3450 return AArch64::LDURSHXi;
3451 case AArch64::LDRSHWui:
3452 return AArch64::LDURSHWi;
3453 case AArch64::LDRBBui:
3454 return AArch64::LDURBBi;
3455 case AArch64::LDRBui:
3456 return AArch64::LDURBi;
3457 case AArch64::STRBBui:
3458 return AArch64::STURBBi;
3459 case AArch64::STRBui:
3460 return AArch64::STURBi;
3461 case AArch64::LDRSBWui:
3462 return AArch64::LDURSBWi;
3463 case AArch64::LDRSBXui:
3464 return AArch64::LDURSBXi;
3477 case AArch64::LDRQroX:
3478 case AArch64::LDURQi:
3479 case AArch64::LDRQui:
3480 return AArch64::LDRQroW;
3481 case AArch64::STRQroX:
3482 case AArch64::STURQi:
3483 case AArch64::STRQui:
3484 return AArch64::STRQroW;
3485 case AArch64::LDRDroX:
3486 case AArch64::LDURDi:
3487 case AArch64::LDRDui:
3488 return AArch64::LDRDroW;
3489 case AArch64::STRDroX:
3490 case AArch64::STURDi:
3491 case AArch64::STRDui:
3492 return AArch64::STRDroW;
3493 case AArch64::LDRXroX:
3494 case AArch64::LDURXi:
3495 case AArch64::LDRXui:
3496 return AArch64::LDRXroW;
3497 case AArch64::STRXroX:
3498 case AArch64::STURXi:
3499 case AArch64::STRXui:
3500 return AArch64::STRXroW;
3501 case AArch64::LDRWroX:
3502 case AArch64::LDURWi:
3503 case AArch64::LDRWui:
3504 return AArch64::LDRWroW;
3505 case AArch64::LDRSWroX:
3506 case AArch64::LDURSWi:
3507 case AArch64::LDRSWui:
3508 return AArch64::LDRSWroW;
3509 case AArch64::STRWroX:
3510 case AArch64::STURWi:
3511 case AArch64::STRWui:
3512 return AArch64::STRWroW;
3513 case AArch64::LDRHroX:
3514 case AArch64::LDURHi:
3515 case AArch64::LDRHui:
3516 return AArch64::LDRHroW;
3517 case AArch64::STRHroX:
3518 case AArch64::STURHi:
3519 case AArch64::STRHui:
3520 return AArch64::STRHroW;
3521 case AArch64::LDRHHroX:
3522 case AArch64::LDURHHi:
3523 case AArch64::LDRHHui:
3524 return AArch64::LDRHHroW;
3525 case AArch64::STRHHroX:
3526 case AArch64::STURHHi:
3527 case AArch64::STRHHui:
3528 return AArch64::STRHHroW;
3529 case AArch64::LDRSHXroX:
3530 case AArch64::LDURSHXi:
3531 case AArch64::LDRSHXui:
3532 return AArch64::LDRSHXroW;
3533 case AArch64::LDRSHWroX:
3534 case AArch64::LDURSHWi:
3535 case AArch64::LDRSHWui:
3536 return AArch64::LDRSHWroW;
3537 case AArch64::LDRBroX:
3538 case AArch64::LDURBi:
3539 case AArch64::LDRBui:
3540 return AArch64::LDRBroW;
3541 case AArch64::LDRBBroX:
3542 case AArch64::LDURBBi:
3543 case AArch64::LDRBBui:
3544 return AArch64::LDRBBroW;
3545 case AArch64::LDRSBXroX:
3546 case AArch64::LDURSBXi:
3547 case AArch64::LDRSBXui:
3548 return AArch64::LDRSBXroW;
3549 case AArch64::LDRSBWroX:
3550 case AArch64::LDURSBWi:
3551 case AArch64::LDRSBWui:
3552 return AArch64::LDRSBWroW;
3553 case AArch64::STRBroX:
3554 case AArch64::STURBi:
3555 case AArch64::STRBui:
3556 return AArch64::STRBroW;
3557 case AArch64::STRBBroX:
3558 case AArch64::STURBBi:
3559 case AArch64::STRBBui:
3560 return AArch64::STRBBroW;
3575 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3585 return B.getInstr();
3589 "Addressing mode not supported for folding");
3606 return B.getInstr();
3613 "Address offset can be a register or an immediate, but not both");
3615 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3620 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3634 return B.getInstr();
3638 "Function must not be called with an addressing mode it can't handle");
3647 case AArch64::LD1Fourv16b_POST:
3648 case AArch64::LD1Fourv1d_POST:
3649 case AArch64::LD1Fourv2d_POST:
3650 case AArch64::LD1Fourv2s_POST:
3651 case AArch64::LD1Fourv4h_POST:
3652 case AArch64::LD1Fourv4s_POST:
3653 case AArch64::LD1Fourv8b_POST:
3654 case AArch64::LD1Fourv8h_POST:
3655 case AArch64::LD1Onev16b_POST:
3656 case AArch64::LD1Onev1d_POST:
3657 case AArch64::LD1Onev2d_POST:
3658 case AArch64::LD1Onev2s_POST:
3659 case AArch64::LD1Onev4h_POST:
3660 case AArch64::LD1Onev4s_POST:
3661 case AArch64::LD1Onev8b_POST:
3662 case AArch64::LD1Onev8h_POST:
3663 case AArch64::LD1Rv16b_POST:
3664 case AArch64::LD1Rv1d_POST:
3665 case AArch64::LD1Rv2d_POST:
3666 case AArch64::LD1Rv2s_POST:
3667 case AArch64::LD1Rv4h_POST:
3668 case AArch64::LD1Rv4s_POST:
3669 case AArch64::LD1Rv8b_POST:
3670 case AArch64::LD1Rv8h_POST:
3671 case AArch64::LD1Threev16b_POST:
3672 case AArch64::LD1Threev1d_POST:
3673 case AArch64::LD1Threev2d_POST:
3674 case AArch64::LD1Threev2s_POST:
3675 case AArch64::LD1Threev4h_POST:
3676 case AArch64::LD1Threev4s_POST:
3677 case AArch64::LD1Threev8b_POST:
3678 case AArch64::LD1Threev8h_POST:
3679 case AArch64::LD1Twov16b_POST:
3680 case AArch64::LD1Twov1d_POST:
3681 case AArch64::LD1Twov2d_POST:
3682 case AArch64::LD1Twov2s_POST:
3683 case AArch64::LD1Twov4h_POST:
3684 case AArch64::LD1Twov4s_POST:
3685 case AArch64::LD1Twov8b_POST:
3686 case AArch64::LD1Twov8h_POST:
3687 case AArch64::LD1i16_POST:
3688 case AArch64::LD1i32_POST:
3689 case AArch64::LD1i64_POST:
3690 case AArch64::LD1i8_POST:
3691 case AArch64::LD2Rv16b_POST:
3692 case AArch64::LD2Rv1d_POST:
3693 case AArch64::LD2Rv2d_POST:
3694 case AArch64::LD2Rv2s_POST:
3695 case AArch64::LD2Rv4h_POST:
3696 case AArch64::LD2Rv4s_POST:
3697 case AArch64::LD2Rv8b_POST:
3698 case AArch64::LD2Rv8h_POST:
3699 case AArch64::LD2Twov16b_POST:
3700 case AArch64::LD2Twov2d_POST:
3701 case AArch64::LD2Twov2s_POST:
3702 case AArch64::LD2Twov4h_POST:
3703 case AArch64::LD2Twov4s_POST:
3704 case AArch64::LD2Twov8b_POST:
3705 case AArch64::LD2Twov8h_POST:
3706 case AArch64::LD2i16_POST:
3707 case AArch64::LD2i32_POST:
3708 case AArch64::LD2i64_POST:
3709 case AArch64::LD2i8_POST:
3710 case AArch64::LD3Rv16b_POST:
3711 case AArch64::LD3Rv1d_POST:
3712 case AArch64::LD3Rv2d_POST:
3713 case AArch64::LD3Rv2s_POST:
3714 case AArch64::LD3Rv4h_POST:
3715 case AArch64::LD3Rv4s_POST:
3716 case AArch64::LD3Rv8b_POST:
3717 case AArch64::LD3Rv8h_POST:
3718 case AArch64::LD3Threev16b_POST:
3719 case AArch64::LD3Threev2d_POST:
3720 case AArch64::LD3Threev2s_POST:
3721 case AArch64::LD3Threev4h_POST:
3722 case AArch64::LD3Threev4s_POST:
3723 case AArch64::LD3Threev8b_POST:
3724 case AArch64::LD3Threev8h_POST:
3725 case AArch64::LD3i16_POST:
3726 case AArch64::LD3i32_POST:
3727 case AArch64::LD3i64_POST:
3728 case AArch64::LD3i8_POST:
3729 case AArch64::LD4Fourv16b_POST:
3730 case AArch64::LD4Fourv2d_POST:
3731 case AArch64::LD4Fourv2s_POST:
3732 case AArch64::LD4Fourv4h_POST:
3733 case AArch64::LD4Fourv4s_POST:
3734 case AArch64::LD4Fourv8b_POST:
3735 case AArch64::LD4Fourv8h_POST:
3736 case AArch64::LD4Rv16b_POST:
3737 case AArch64::LD4Rv1d_POST:
3738 case AArch64::LD4Rv2d_POST:
3739 case AArch64::LD4Rv2s_POST:
3740 case AArch64::LD4Rv4h_POST:
3741 case AArch64::LD4Rv4s_POST:
3742 case AArch64::LD4Rv8b_POST:
3743 case AArch64::LD4Rv8h_POST:
3744 case AArch64::LD4i16_POST:
3745 case AArch64::LD4i32_POST:
3746 case AArch64::LD4i64_POST:
3747 case AArch64::LD4i8_POST:
3748 case AArch64::LDAPRWpost:
3749 case AArch64::LDAPRXpost:
3750 case AArch64::LDIAPPWpost:
3751 case AArch64::LDIAPPXpost:
3752 case AArch64::LDPDpost:
3753 case AArch64::LDPQpost:
3754 case AArch64::LDPSWpost:
3755 case AArch64::LDPSpost:
3756 case AArch64::LDPWpost:
3757 case AArch64::LDPXpost:
3758 case AArch64::LDRBBpost:
3759 case AArch64::LDRBpost:
3760 case AArch64::LDRDpost:
3761 case AArch64::LDRHHpost:
3762 case AArch64::LDRHpost:
3763 case AArch64::LDRQpost:
3764 case AArch64::LDRSBWpost:
3765 case AArch64::LDRSBXpost:
3766 case AArch64::LDRSHWpost:
3767 case AArch64::LDRSHXpost:
3768 case AArch64::LDRSWpost:
3769 case AArch64::LDRSpost:
3770 case AArch64::LDRWpost:
3771 case AArch64::LDRXpost:
3772 case AArch64::ST1Fourv16b_POST:
3773 case AArch64::ST1Fourv1d_POST:
3774 case AArch64::ST1Fourv2d_POST:
3775 case AArch64::ST1Fourv2s_POST:
3776 case AArch64::ST1Fourv4h_POST:
3777 case AArch64::ST1Fourv4s_POST:
3778 case AArch64::ST1Fourv8b_POST:
3779 case AArch64::ST1Fourv8h_POST:
3780 case AArch64::ST1Onev16b_POST:
3781 case AArch64::ST1Onev1d_POST:
3782 case AArch64::ST1Onev2d_POST:
3783 case AArch64::ST1Onev2s_POST:
3784 case AArch64::ST1Onev4h_POST:
3785 case AArch64::ST1Onev4s_POST:
3786 case AArch64::ST1Onev8b_POST:
3787 case AArch64::ST1Onev8h_POST:
3788 case AArch64::ST1Threev16b_POST:
3789 case AArch64::ST1Threev1d_POST:
3790 case AArch64::ST1Threev2d_POST:
3791 case AArch64::ST1Threev2s_POST:
3792 case AArch64::ST1Threev4h_POST:
3793 case AArch64::ST1Threev4s_POST:
3794 case AArch64::ST1Threev8b_POST:
3795 case AArch64::ST1Threev8h_POST:
3796 case AArch64::ST1Twov16b_POST:
3797 case AArch64::ST1Twov1d_POST:
3798 case AArch64::ST1Twov2d_POST:
3799 case AArch64::ST1Twov2s_POST:
3800 case AArch64::ST1Twov4h_POST:
3801 case AArch64::ST1Twov4s_POST:
3802 case AArch64::ST1Twov8b_POST:
3803 case AArch64::ST1Twov8h_POST:
3804 case AArch64::ST1i16_POST:
3805 case AArch64::ST1i32_POST:
3806 case AArch64::ST1i64_POST:
3807 case AArch64::ST1i8_POST:
3808 case AArch64::ST2GPostIndex:
3809 case AArch64::ST2Twov16b_POST:
3810 case AArch64::ST2Twov2d_POST:
3811 case AArch64::ST2Twov2s_POST:
3812 case AArch64::ST2Twov4h_POST:
3813 case AArch64::ST2Twov4s_POST:
3814 case AArch64::ST2Twov8b_POST:
3815 case AArch64::ST2Twov8h_POST:
3816 case AArch64::ST2i16_POST:
3817 case AArch64::ST2i32_POST:
3818 case AArch64::ST2i64_POST:
3819 case AArch64::ST2i8_POST:
3820 case AArch64::ST3Threev16b_POST:
3821 case AArch64::ST3Threev2d_POST:
3822 case AArch64::ST3Threev2s_POST:
3823 case AArch64::ST3Threev4h_POST:
3824 case AArch64::ST3Threev4s_POST:
3825 case AArch64::ST3Threev8b_POST:
3826 case AArch64::ST3Threev8h_POST:
3827 case AArch64::ST3i16_POST:
3828 case AArch64::ST3i32_POST:
3829 case AArch64::ST3i64_POST:
3830 case AArch64::ST3i8_POST:
3831 case AArch64::ST4Fourv16b_POST:
3832 case AArch64::ST4Fourv2d_POST:
3833 case AArch64::ST4Fourv2s_POST:
3834 case AArch64::ST4Fourv4h_POST:
3835 case AArch64::ST4Fourv4s_POST:
3836 case AArch64::ST4Fourv8b_POST:
3837 case AArch64::ST4Fourv8h_POST:
3838 case AArch64::ST4i16_POST:
3839 case AArch64::ST4i32_POST:
3840 case AArch64::ST4i64_POST:
3841 case AArch64::ST4i8_POST:
3842 case AArch64::STGPostIndex:
3843 case AArch64::STGPpost:
3844 case AArch64::STPDpost:
3845 case AArch64::STPQpost:
3846 case AArch64::STPSpost:
3847 case AArch64::STPWpost:
3848 case AArch64::STPXpost:
3849 case AArch64::STRBBpost:
3850 case AArch64::STRBpost:
3851 case AArch64::STRDpost:
3852 case AArch64::STRHHpost:
3853 case AArch64::STRHpost:
3854 case AArch64::STRQpost:
3855 case AArch64::STRSpost:
3856 case AArch64::STRWpost:
3857 case AArch64::STRXpost:
3858 case AArch64::STZ2GPostIndex:
3859 case AArch64::STZGPostIndex:
3866 bool &OffsetIsScalable,
TypeSize &Width,
3887 int64_t Dummy1, Dummy2;
3909 return BaseOp->
isReg() || BaseOp->
isFI();
3916 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3921 TypeSize &Width, int64_t &MinOffset,
3922 int64_t &MaxOffset) {
3928 MinOffset = MaxOffset = 0;
3931 case AArch64::LDRQui:
3932 case AArch64::STRQui:
3938 case AArch64::LDRXui:
3939 case AArch64::LDRDui:
3940 case AArch64::STRXui:
3941 case AArch64::STRDui:
3942 case AArch64::PRFMui:
3948 case AArch64::LDRWui:
3949 case AArch64::LDRSui:
3950 case AArch64::LDRSWui:
3951 case AArch64::STRWui:
3952 case AArch64::STRSui:
3958 case AArch64::LDRHui:
3959 case AArch64::LDRHHui:
3960 case AArch64::LDRSHWui:
3961 case AArch64::LDRSHXui:
3962 case AArch64::STRHui:
3963 case AArch64::STRHHui:
3969 case AArch64::LDRBui:
3970 case AArch64::LDRBBui:
3971 case AArch64::LDRSBWui:
3972 case AArch64::LDRSBXui:
3973 case AArch64::STRBui:
3974 case AArch64::STRBBui:
3981 case AArch64::STRQpre:
3982 case AArch64::LDRQpost:
3988 case AArch64::LDRDpost:
3989 case AArch64::LDRDpre:
3990 case AArch64::LDRXpost:
3991 case AArch64::LDRXpre:
3992 case AArch64::STRDpost:
3993 case AArch64::STRDpre:
3994 case AArch64::STRXpost:
3995 case AArch64::STRXpre:
4001 case AArch64::STRWpost:
4002 case AArch64::STRWpre:
4003 case AArch64::LDRWpost:
4004 case AArch64::LDRWpre:
4005 case AArch64::STRSpost:
4006 case AArch64::STRSpre:
4007 case AArch64::LDRSpost:
4008 case AArch64::LDRSpre:
4014 case AArch64::LDRHpost:
4015 case AArch64::LDRHpre:
4016 case AArch64::STRHpost:
4017 case AArch64::STRHpre:
4018 case AArch64::LDRHHpost:
4019 case AArch64::LDRHHpre:
4020 case AArch64::STRHHpost:
4021 case AArch64::STRHHpre:
4027 case AArch64::LDRBpost:
4028 case AArch64::LDRBpre:
4029 case AArch64::STRBpost:
4030 case AArch64::STRBpre:
4031 case AArch64::LDRBBpost:
4032 case AArch64::LDRBBpre:
4033 case AArch64::STRBBpost:
4034 case AArch64::STRBBpre:
4041 case AArch64::LDURQi:
4042 case AArch64::STURQi:
4048 case AArch64::LDURXi:
4049 case AArch64::LDURDi:
4050 case AArch64::LDAPURXi:
4051 case AArch64::STURXi:
4052 case AArch64::STURDi:
4053 case AArch64::STLURXi:
4054 case AArch64::PRFUMi:
4060 case AArch64::LDURWi:
4061 case AArch64::LDURSi:
4062 case AArch64::LDURSWi:
4063 case AArch64::LDAPURi:
4064 case AArch64::LDAPURSWi:
4065 case AArch64::STURWi:
4066 case AArch64::STURSi:
4067 case AArch64::STLURWi:
4073 case AArch64::LDURHi:
4074 case AArch64::LDURHHi:
4075 case AArch64::LDURSHXi:
4076 case AArch64::LDURSHWi:
4077 case AArch64::LDAPURHi:
4078 case AArch64::LDAPURSHWi:
4079 case AArch64::LDAPURSHXi:
4080 case AArch64::STURHi:
4081 case AArch64::STURHHi:
4082 case AArch64::STLURHi:
4088 case AArch64::LDURBi:
4089 case AArch64::LDURBBi:
4090 case AArch64::LDURSBXi:
4091 case AArch64::LDURSBWi:
4092 case AArch64::LDAPURBi:
4093 case AArch64::LDAPURSBWi:
4094 case AArch64::LDAPURSBXi:
4095 case AArch64::STURBi:
4096 case AArch64::STURBBi:
4097 case AArch64::STLURBi:
4104 case AArch64::LDPQi:
4105 case AArch64::LDNPQi:
4106 case AArch64::STPQi:
4107 case AArch64::STNPQi:
4108 case AArch64::LDPQpost:
4109 case AArch64::LDPQpre:
4110 case AArch64::STPQpost:
4111 case AArch64::STPQpre:
4117 case AArch64::LDPXi:
4118 case AArch64::LDPDi:
4119 case AArch64::LDNPXi:
4120 case AArch64::LDNPDi:
4121 case AArch64::STPXi:
4122 case AArch64::STPDi:
4123 case AArch64::STNPXi:
4124 case AArch64::STNPDi:
4125 case AArch64::LDPDpost:
4126 case AArch64::LDPDpre:
4127 case AArch64::LDPXpost:
4128 case AArch64::LDPXpre:
4129 case AArch64::STPDpost:
4130 case AArch64::STPDpre:
4131 case AArch64::STPXpost:
4132 case AArch64::STPXpre:
4138 case AArch64::LDPWi:
4139 case AArch64::LDPSi:
4140 case AArch64::LDNPWi:
4141 case AArch64::LDNPSi:
4142 case AArch64::STPWi:
4143 case AArch64::STPSi:
4144 case AArch64::STNPWi:
4145 case AArch64::STNPSi:
4146 case AArch64::LDPSpost:
4147 case AArch64::LDPSpre:
4148 case AArch64::LDPWpost:
4149 case AArch64::LDPWpre:
4150 case AArch64::STPSpost:
4151 case AArch64::STPSpre:
4152 case AArch64::STPWpost:
4153 case AArch64::STPWpre:
4159 case AArch64::StoreSwiftAsyncContext:
4172 case AArch64::TAGPstack:
4182 case AArch64::STGPreIndex:
4183 case AArch64::STGPostIndex:
4184 case AArch64::STZGi:
4185 case AArch64::STZGPreIndex:
4186 case AArch64::STZGPostIndex:
4193 case AArch64::STR_ZZZZXI:
4194 case AArch64::LDR_ZZZZXI:
4200 case AArch64::STR_ZZZXI:
4201 case AArch64::LDR_ZZZXI:
4207 case AArch64::STR_ZZXI:
4208 case AArch64::LDR_ZZXI:
4214 case AArch64::LDR_PXI:
4215 case AArch64::STR_PXI:
4221 case AArch64::LDR_PPXI:
4222 case AArch64::STR_PPXI:
4228 case AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO:
4229 case AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO:
4230 case AArch64::LDR_ZXI:
4231 case AArch64::STR_ZXI:
4237 case AArch64::LD1B_IMM:
4238 case AArch64::LD1H_IMM:
4239 case AArch64::LD1W_IMM:
4240 case AArch64::LD1D_IMM:
4241 case AArch64::LDNT1B_ZRI:
4242 case AArch64::LDNT1H_ZRI:
4243 case AArch64::LDNT1W_ZRI:
4244 case AArch64::LDNT1D_ZRI:
4245 case AArch64::ST1B_IMM:
4246 case AArch64::ST1H_IMM:
4247 case AArch64::ST1W_IMM:
4248 case AArch64::ST1D_IMM:
4249 case AArch64::STNT1B_ZRI:
4250 case AArch64::STNT1H_ZRI:
4251 case AArch64::STNT1W_ZRI:
4252 case AArch64::STNT1D_ZRI:
4253 case AArch64::LDNF1B_IMM:
4254 case AArch64::LDNF1H_IMM:
4255 case AArch64::LDNF1W_IMM:
4256 case AArch64::LDNF1D_IMM:
4264 case AArch64::LD2B_IMM:
4265 case AArch64::LD2H_IMM:
4266 case AArch64::LD2W_IMM:
4267 case AArch64::LD2D_IMM:
4268 case AArch64::ST2B_IMM:
4269 case AArch64::ST2H_IMM:
4270 case AArch64::ST2W_IMM:
4271 case AArch64::ST2D_IMM:
4277 case AArch64::LD3B_IMM:
4278 case AArch64::LD3H_IMM:
4279 case AArch64::LD3W_IMM:
4280 case AArch64::LD3D_IMM:
4281 case AArch64::ST3B_IMM:
4282 case AArch64::ST3H_IMM:
4283 case AArch64::ST3W_IMM:
4284 case AArch64::ST3D_IMM:
4290 case AArch64::LD4B_IMM:
4291 case AArch64::LD4H_IMM:
4292 case AArch64::LD4W_IMM:
4293 case AArch64::LD4D_IMM:
4294 case AArch64::ST4B_IMM:
4295 case AArch64::ST4H_IMM:
4296 case AArch64::ST4W_IMM:
4297 case AArch64::ST4D_IMM:
4303 case AArch64::LD1B_H_IMM:
4304 case AArch64::LD1SB_H_IMM:
4305 case AArch64::LD1H_S_IMM:
4306 case AArch64::LD1SH_S_IMM:
4307 case AArch64::LD1W_D_IMM:
4308 case AArch64::LD1SW_D_IMM:
4309 case AArch64::ST1B_H_IMM:
4310 case AArch64::ST1H_S_IMM:
4311 case AArch64::ST1W_D_IMM:
4312 case AArch64::LDNF1B_H_IMM:
4313 case AArch64::LDNF1SB_H_IMM:
4314 case AArch64::LDNF1H_S_IMM:
4315 case AArch64::LDNF1SH_S_IMM:
4316 case AArch64::LDNF1W_D_IMM:
4317 case AArch64::LDNF1SW_D_IMM:
4325 case AArch64::LD1B_S_IMM:
4326 case AArch64::LD1SB_S_IMM:
4327 case AArch64::LD1H_D_IMM:
4328 case AArch64::LD1SH_D_IMM:
4329 case AArch64::ST1B_S_IMM:
4330 case AArch64::ST1H_D_IMM:
4331 case AArch64::LDNF1B_S_IMM:
4332 case AArch64::LDNF1SB_S_IMM:
4333 case AArch64::LDNF1H_D_IMM:
4334 case AArch64::LDNF1SH_D_IMM:
4342 case AArch64::LD1B_D_IMM:
4343 case AArch64::LD1SB_D_IMM:
4344 case AArch64::ST1B_D_IMM:
4345 case AArch64::LDNF1B_D_IMM:
4346 case AArch64::LDNF1SB_D_IMM:
4354 case AArch64::ST2Gi:
4355 case AArch64::ST2GPreIndex:
4356 case AArch64::ST2GPostIndex:
4357 case AArch64::STZ2Gi:
4358 case AArch64::STZ2GPreIndex:
4359 case AArch64::STZ2GPostIndex:
4365 case AArch64::STGPi:
4366 case AArch64::STGPpost:
4367 case AArch64::STGPpre:
4373 case AArch64::LD1RB_IMM:
4374 case AArch64::LD1RB_H_IMM:
4375 case AArch64::LD1RB_S_IMM:
4376 case AArch64::LD1RB_D_IMM:
4377 case AArch64::LD1RSB_H_IMM:
4378 case AArch64::LD1RSB_S_IMM:
4379 case AArch64::LD1RSB_D_IMM:
4385 case AArch64::LD1RH_IMM:
4386 case AArch64::LD1RH_S_IMM:
4387 case AArch64::LD1RH_D_IMM:
4388 case AArch64::LD1RSH_S_IMM:
4389 case AArch64::LD1RSH_D_IMM:
4395 case AArch64::LD1RW_IMM:
4396 case AArch64::LD1RW_D_IMM:
4397 case AArch64::LD1RSW_IMM:
4403 case AArch64::LD1RD_IMM:
4419 case AArch64::LDRBBui:
4420 case AArch64::LDURBBi:
4421 case AArch64::LDRSBWui:
4422 case AArch64::LDURSBWi:
4423 case AArch64::STRBBui:
4424 case AArch64::STURBBi:
4426 case AArch64::LDRHHui:
4427 case AArch64::LDURHHi:
4428 case AArch64::LDRSHWui:
4429 case AArch64::LDURSHWi:
4430 case AArch64::STRHHui:
4431 case AArch64::STURHHi:
4433 case AArch64::LDRSui:
4434 case AArch64::LDURSi:
4435 case AArch64::LDRSpre:
4436 case AArch64::LDRSWui:
4437 case AArch64::LDURSWi:
4438 case AArch64::LDRSWpre:
4439 case AArch64::LDRWpre:
4440 case AArch64::LDRWui:
4441 case AArch64::LDURWi:
4442 case AArch64::STRSui:
4443 case AArch64::STURSi:
4444 case AArch64::STRSpre:
4445 case AArch64::STRWui:
4446 case AArch64::STURWi:
4447 case AArch64::STRWpre:
4448 case AArch64::LDPSi:
4449 case AArch64::LDPSWi:
4450 case AArch64::LDPWi:
4451 case AArch64::STPSi:
4452 case AArch64::STPWi:
4454 case AArch64::LDRDui:
4455 case AArch64::LDURDi:
4456 case AArch64::LDRDpre:
4457 case AArch64::LDRXui:
4458 case AArch64::LDURXi:
4459 case AArch64::LDRXpre:
4460 case AArch64::STRDui:
4461 case AArch64::STURDi:
4462 case AArch64::STRDpre:
4463 case AArch64::STRXui:
4464 case AArch64::STURXi:
4465 case AArch64::STRXpre:
4466 case AArch64::LDPDi:
4467 case AArch64::LDPXi:
4468 case AArch64::STPDi:
4469 case AArch64::STPXi:
4471 case AArch64::LDRQui:
4472 case AArch64::LDURQi:
4473 case AArch64::STRQui:
4474 case AArch64::STURQi:
4475 case AArch64::STRQpre:
4476 case AArch64::LDPQi:
4477 case AArch64::LDRQpre:
4478 case AArch64::STPQi:
4480 case AArch64::STZGi:
4481 case AArch64::ST2Gi:
4482 case AArch64::STZ2Gi:
4483 case AArch64::STGPi:
4489 switch (
MI.getOpcode()) {
4492 case AArch64::LDRWpre:
4493 case AArch64::LDRXpre:
4494 case AArch64::LDRSWpre:
4495 case AArch64::LDRSpre:
4496 case AArch64::LDRDpre:
4497 case AArch64::LDRQpre:
4503 switch (
MI.getOpcode()) {
4506 case AArch64::STRWpre:
4507 case AArch64::STRXpre:
4508 case AArch64::STRSpre:
4509 case AArch64::STRDpre:
4510 case AArch64::STRQpre:
4520 switch (
MI.getOpcode()) {
4523 case AArch64::LDPSi:
4524 case AArch64::LDPSWi:
4525 case AArch64::LDPDi:
4526 case AArch64::LDPQi:
4527 case AArch64::LDPWi:
4528 case AArch64::LDPXi:
4529 case AArch64::STPSi:
4530 case AArch64::STPDi:
4531 case AArch64::STPQi:
4532 case AArch64::STPWi:
4533 case AArch64::STPXi:
4534 case AArch64::STGPi:
4540 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4544 return MI.getOperand(
Idx);
4549 assert(
MI.mayLoadOrStore() &&
"Load or store instruction expected");
4553 return MI.getOperand(
Idx);
4558 switch (
MI.getOpcode()) {
4561 case AArch64::LDRBroX:
4562 case AArch64::LDRBBroX:
4563 case AArch64::LDRSBXroX:
4564 case AArch64::LDRSBWroX:
4565 case AArch64::LDRHroX:
4566 case AArch64::LDRHHroX:
4567 case AArch64::LDRSHXroX:
4568 case AArch64::LDRSHWroX:
4569 case AArch64::LDRWroX:
4570 case AArch64::LDRSroX:
4571 case AArch64::LDRSWroX:
4572 case AArch64::LDRDroX:
4573 case AArch64::LDRXroX:
4574 case AArch64::LDRQroX:
4575 return MI.getOperand(4);
4581 if (
MI.getParent() ==
nullptr)
4591 auto Reg =
Op.getReg();
4592 if (Reg.isPhysical())
4593 return AArch64::FPR16RegClass.
contains(Reg);
4595 return TRC == &AArch64::FPR16RegClass ||
4596 TRC == &AArch64::FPR16_loRegClass;
4605 auto Reg =
Op.getReg();
4606 if (Reg.isPhysical())
4607 return AArch64::FPR128RegClass.
contains(Reg);
4609 return TRC == &AArch64::FPR128RegClass ||
4610 TRC == &AArch64::FPR128_loRegClass;
4616 switch (
MI.getOpcode()) {
4619 case AArch64::PACIASP:
4620 case AArch64::PACIBSP:
4623 case AArch64::PAUTH_PROLOGUE:
4626 case AArch64::HINT: {
4627 unsigned Imm =
MI.getOperand(0).getImm();
4629 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4632 if (Imm == 25 || Imm == 27)
4644 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4645 return AArch64::FPR128RegClass.contains(Reg) ||
4646 AArch64::FPR64RegClass.contains(Reg) ||
4647 AArch64::FPR32RegClass.contains(Reg) ||
4648 AArch64::FPR16RegClass.contains(Reg) ||
4649 AArch64::FPR8RegClass.contains(Reg);
4656 auto Reg =
Op.getReg();
4657 if (Reg.isPhysical())
4661 return TRC == &AArch64::FPR128RegClass ||
4662 TRC == &AArch64::FPR128_loRegClass ||
4663 TRC == &AArch64::FPR64RegClass ||
4664 TRC == &AArch64::FPR64_loRegClass ||
4665 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4666 TRC == &AArch64::FPR8RegClass;
4688 if (FirstOpc == SecondOpc)
4694 case AArch64::STRSui:
4695 case AArch64::STURSi:
4696 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4697 case AArch64::STRDui:
4698 case AArch64::STURDi:
4699 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4700 case AArch64::STRQui:
4701 case AArch64::STURQi:
4702 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4703 case AArch64::STRWui:
4704 case AArch64::STURWi:
4705 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4706 case AArch64::STRXui:
4707 case AArch64::STURXi:
4708 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4709 case AArch64::LDRSui:
4710 case AArch64::LDURSi:
4711 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4712 case AArch64::LDRDui:
4713 case AArch64::LDURDi:
4714 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4715 case AArch64::LDRQui:
4716 case AArch64::LDURQi:
4717 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4718 case AArch64::LDRWui:
4719 case AArch64::LDURWi:
4720 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4721 case AArch64::LDRSWui:
4722 case AArch64::LDURSWi:
4723 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4724 case AArch64::LDRXui:
4725 case AArch64::LDURXi:
4726 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4733 int64_t Offset1,
unsigned Opcode1,
int FI2,
4734 int64_t Offset2,
unsigned Opcode2) {
4740 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4743 if (ObjectOffset1 % Scale1 != 0)
4745 ObjectOffset1 /= Scale1;
4747 if (ObjectOffset2 % Scale2 != 0)
4749 ObjectOffset2 /= Scale2;
4750 ObjectOffset1 += Offset1;
4751 ObjectOffset2 += Offset2;
4752 return ObjectOffset1 + 1 == ObjectOffset2;
4764 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4765 unsigned NumBytes)
const {
4775 "Only base registers and frame indices are supported.");
4782 if (ClusterSize > 2)
4789 unsigned FirstOpc = FirstLdSt.
getOpcode();
4790 unsigned SecondOpc = SecondLdSt.
getOpcode();
4810 if (Offset1 > 63 || Offset1 < -64)
4815 if (BaseOp1.
isFI()) {
4817 "Caller should have ordered offsets.");
4822 BaseOp2.
getIndex(), Offset2, SecondOpc);
4825 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4827 return Offset1 + 1 == Offset2;
4835 return MIB.
addReg(Reg, State);
4837 if (Reg.isPhysical())
4838 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4839 return MIB.
addReg(Reg, State, SubIdx);
4846 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4855 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4857 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4858 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4859 unsigned NumRegs = Indices.
size();
4861 int SubReg = 0,
End = NumRegs, Incr = 1;
4880 unsigned Opcode,
unsigned ZeroReg,
4883 unsigned NumRegs = Indices.
size();
4886 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4887 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4888 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4889 "GPR reg sequences should not be able to overlap");
4906 bool RenamableSrc)
const {
4907 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4908 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4911 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4913 if (Subtarget.hasZeroCycleRegMove()) {
4916 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4918 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4934 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4939 if (Subtarget.hasZeroCycleRegMove()) {
4942 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4944 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4964 if (AArch64::PPRRegClass.
contains(DestReg) &&
4965 AArch64::PPRRegClass.
contains(SrcReg)) {
4967 "Unexpected SVE register.");
4977 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4978 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4979 if (DestIsPNR || SrcIsPNR) {
4981 return (R - AArch64::PN0) + AArch64::P0;
4983 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4984 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4986 if (PPRSrcReg != PPRDestReg) {
4998 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4999 AArch64::ZPRRegClass.
contains(SrcReg)) {
5001 "Unexpected SVE register.");
5009 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
5010 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
5011 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
5012 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
5014 "Unexpected SVE register.");
5015 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5022 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
5023 AArch64::ZPR3RegClass.
contains(SrcReg)) {
5025 "Unexpected SVE register.");
5026 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5034 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
5035 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
5036 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
5037 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
5039 "Unexpected SVE register.");
5040 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5041 AArch64::zsub2, AArch64::zsub3};
5047 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
5048 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
5049 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5055 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
5069 if (AArch64::DDDDRegClass.
contains(DestReg) &&
5070 AArch64::DDDDRegClass.
contains(SrcReg)) {
5071 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5072 AArch64::dsub2, AArch64::dsub3};
5079 if (AArch64::DDDRegClass.
contains(DestReg) &&
5080 AArch64::DDDRegClass.
contains(SrcReg)) {
5081 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
5089 if (AArch64::DDRegClass.
contains(DestReg) &&
5090 AArch64::DDRegClass.
contains(SrcReg)) {
5091 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
5098 if (AArch64::QQQQRegClass.
contains(DestReg) &&
5099 AArch64::QQQQRegClass.
contains(SrcReg)) {
5100 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5101 AArch64::qsub2, AArch64::qsub3};
5108 if (AArch64::QQQRegClass.
contains(DestReg) &&
5109 AArch64::QQQRegClass.
contains(SrcReg)) {
5110 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
5118 if (AArch64::QQRegClass.
contains(DestReg) &&
5119 AArch64::QQRegClass.
contains(SrcReg)) {
5120 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
5126 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
5127 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
5128 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
5130 AArch64::XZR, Indices);
5134 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
5135 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
5136 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
5138 AArch64::WZR, Indices);
5142 if (AArch64::FPR128RegClass.
contains(DestReg) &&
5143 AArch64::FPR128RegClass.
contains(SrcReg)) {
5148 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
5149 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
5169 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5170 AArch64::FPR64RegClass.
contains(SrcReg)) {
5176 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5177 AArch64::FPR32RegClass.
contains(SrcReg)) {
5183 if (AArch64::FPR16RegClass.
contains(DestReg) &&
5184 AArch64::FPR16RegClass.
contains(SrcReg)) {
5186 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
5188 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
5194 if (AArch64::FPR8RegClass.
contains(DestReg) &&
5195 AArch64::FPR8RegClass.
contains(SrcReg)) {
5197 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
5199 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
5206 if (AArch64::FPR64RegClass.
contains(DestReg) &&
5207 AArch64::GPR64RegClass.
contains(SrcReg)) {
5212 if (AArch64::GPR64RegClass.
contains(DestReg) &&
5213 AArch64::FPR64RegClass.
contains(SrcReg)) {
5219 if (AArch64::FPR32RegClass.
contains(DestReg) &&
5220 AArch64::GPR32RegClass.
contains(SrcReg)) {
5225 if (AArch64::GPR32RegClass.
contains(DestReg) &&
5226 AArch64::FPR32RegClass.
contains(SrcReg)) {
5232 if (DestReg == AArch64::NZCV) {
5233 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
5235 .
addImm(AArch64SysReg::NZCV)
5241 if (SrcReg == AArch64::NZCV) {
5242 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
5244 .
addImm(AArch64SysReg::NZCV)
5251 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
5252 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
5262 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
5267 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
5269 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
5282 Register SrcReg,
bool isKill,
int FI,
5298 switch (
TRI->getSpillSize(*RC)) {
5300 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5301 Opc = AArch64::STRBui;
5304 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5305 Opc = AArch64::STRHui;
5306 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
5307 AArch64::PPRRegClass.hasSubClassEq(RC)) {
5309 "Unexpected register store without SVE store instructions");
5310 Opc = AArch64::STR_PXI;
5316 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5317 Opc = AArch64::STRWui;
5321 assert(SrcReg != AArch64::WSP);
5322 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5323 Opc = AArch64::STRSui;
5324 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5325 Opc = AArch64::STR_PPXI;
5330 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5331 Opc = AArch64::STRXui;
5335 assert(SrcReg != AArch64::SP);
5336 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5337 Opc = AArch64::STRDui;
5338 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5340 get(AArch64::STPWi), SrcReg, isKill,
5341 AArch64::sube32, AArch64::subo32, FI, MMO);
5346 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5347 Opc = AArch64::STRQui;
5348 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5349 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5350 Opc = AArch64::ST1Twov1d;
5352 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5354 get(AArch64::STPXi), SrcReg, isKill,
5355 AArch64::sube64, AArch64::subo64, FI, MMO);
5357 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5359 "Unexpected register store without SVE store instructions");
5360 Opc = AArch64::STR_ZXI;
5362 }
else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
5364 "Unexpected predicate store without SVE store instructions");
5365 Opc = AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO;
5370 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5371 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5372 Opc = AArch64::ST1Threev1d;
5377 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5378 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5379 Opc = AArch64::ST1Fourv1d;
5381 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5382 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5383 Opc = AArch64::ST1Twov2d;
5385 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5386 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5388 "Unexpected register store without SVE store instructions");
5389 Opc = AArch64::STR_ZZXI;
5394 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5395 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5396 Opc = AArch64::ST1Threev2d;
5398 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5400 "Unexpected register store without SVE store instructions");
5401 Opc = AArch64::STR_ZZZXI;
5406 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5407 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
5408 Opc = AArch64::ST1Fourv2d;
5410 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5411 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5413 "Unexpected register store without SVE store instructions");
5414 Opc = AArch64::STR_ZZZZXI;
5419 assert(Opc &&
"Unknown register class");
5430 MI.addMemOperand(MMO);
5437 Register DestReg,
unsigned SubIdx0,
5438 unsigned SubIdx1,
int FI,
5442 bool IsUndef =
true;
5444 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
5446 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5473 switch (
TRI->getSpillSize(*RC)) {
5475 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5476 Opc = AArch64::LDRBui;
5479 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5480 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5481 Opc = AArch64::LDRHui;
5482 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5484 "Unexpected register load without SVE load instructions");
5487 Opc = AArch64::LDR_PXI;
5493 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5494 Opc = AArch64::LDRWui;
5498 assert(DestReg != AArch64::WSP);
5499 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5500 Opc = AArch64::LDRSui;
5501 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5502 Opc = AArch64::LDR_PPXI;
5507 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5508 Opc = AArch64::LDRXui;
5512 assert(DestReg != AArch64::SP);
5513 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5514 Opc = AArch64::LDRDui;
5515 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5517 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5518 AArch64::subo32, FI, MMO);
5523 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5524 Opc = AArch64::LDRQui;
5525 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5526 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5527 Opc = AArch64::LD1Twov1d;
5529 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5531 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5532 AArch64::subo64, FI, MMO);
5534 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5536 "Unexpected register load without SVE load instructions");
5537 Opc = AArch64::LDR_ZXI;
5539 }
else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
5541 "Unexpected predicate load without SVE load instructions");
5542 Opc = AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO;
5547 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5548 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5549 Opc = AArch64::LD1Threev1d;
5554 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5555 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5556 Opc = AArch64::LD1Fourv1d;
5558 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5559 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5560 Opc = AArch64::LD1Twov2d;
5562 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5563 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5565 "Unexpected register load without SVE load instructions");
5566 Opc = AArch64::LDR_ZZXI;
5571 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5572 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5573 Opc = AArch64::LD1Threev2d;
5575 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5577 "Unexpected register load without SVE load instructions");
5578 Opc = AArch64::LDR_ZZZXI;
5583 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5584 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5585 Opc = AArch64::LD1Fourv2d;
5587 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5588 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5590 "Unexpected register load without SVE load instructions");
5591 Opc = AArch64::LDR_ZZZZXI;
5597 assert(Opc &&
"Unknown register class");
5607 MI.addMemOperand(MMO);
5614 UseMI.getIterator()),
5616 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5617 I.readsRegister(AArch64::NZCV, TRI);
5626 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5633 ByteSized =
Offset.getFixed();
5634 VGSized =
Offset.getScalable() / 2;
5642 int64_t &NumDataVectors) {
5646 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5648 NumBytes =
Offset.getFixed();
5650 NumPredicateVectors =
Offset.getScalable() / 2;
5655 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5656 NumPredicateVectors > 62) {
5657 NumDataVectors = NumPredicateVectors / 8;
5658 NumPredicateVectors -= NumDataVectors * 8;
5665 int NumVGScaledBytes,
unsigned VG,
5673 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5676 if (NumVGScaledBytes) {
5687 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5688 << std::abs(NumVGScaledBytes) <<
" * VG";
5697 int64_t NumBytes, NumVGScaledBytes;
5700 std::string CommentBuffer;
5703 if (Reg == AArch64::SP)
5705 else if (Reg == AArch64::FP)
5712 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5716 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5720 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
5729 unsigned FrameReg,
unsigned Reg,
5731 bool LastAdjustmentWasScalable) {
5732 if (
Offset.getScalable())
5735 if (FrameReg == Reg && !LastAdjustmentWasScalable)
5738 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5745 int64_t NumBytes, NumVGScaledBytes;
5747 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
5749 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5752 if (!NumVGScaledBytes)
5755 std::string CommentBuffer;
5762 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5766 CfaExpr.
push_back(dwarf::DW_CFA_expression);
5782 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
5785 bool *HasWinCFI,
bool EmitCFAOffset,
5788 unsigned MaxEncoding, ShiftSize;
5790 case AArch64::ADDXri:
5791 case AArch64::ADDSXri:
5792 case AArch64::SUBXri:
5793 case AArch64::SUBSXri:
5794 MaxEncoding = 0xfff;
5797 case AArch64::ADDVL_XXI:
5798 case AArch64::ADDPL_XXI:
5799 case AArch64::ADDSVL_XXI:
5800 case AArch64::ADDSPL_XXI:
5815 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
5817 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
5831 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
5833 if (TmpReg == AArch64::XZR)
5835 &AArch64::GPR64RegClass);
5837 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
5838 unsigned LocalShiftSize = 0;
5839 if (ThisVal > MaxEncoding) {
5840 ThisVal = ThisVal >> ShiftSize;
5841 LocalShiftSize = ShiftSize;
5843 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
5844 "Encoding cannot handle value that big");
5846 Offset -= ThisVal << LocalShiftSize;
5851 .
addImm(Sign * (
int)ThisVal);
5861 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
5862 CFAOffset += Change;
5864 CFAOffset -= Change;
5865 if (EmitCFAOffset && DestReg == TmpReg) {
5878 assert(Sign == 1 &&
"SEH directives should always have a positive sign");
5879 int Imm = (int)(ThisVal << LocalShiftSize);
5880 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
5881 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
5890 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
5891 "emit a single SEH directive");
5892 }
else if (DestReg == AArch64::SP) {
5895 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
5908 unsigned DestReg,
unsigned SrcReg,
5911 bool NeedsWinCFI,
bool *HasWinCFI,
5913 unsigned FrameReg) {
5920 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
5922 int64_t Bytes, NumPredicateVectors, NumDataVectors;
5924 Offset, Bytes, NumPredicateVectors, NumDataVectors);
5927 if (Bytes || (!
Offset && SrcReg != DestReg)) {
5928 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
5929 "SP increment/decrement not 8-byte aligned");
5930 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
5933 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
5936 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
5938 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
5945 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
5946 "SetNZCV not supported with SVE vectors");
5947 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
5948 "WinCFI not supported with SVE vectors");
5950 if (NumDataVectors) {
5952 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
5953 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5954 CFAOffset, FrameReg);
5959 if (NumPredicateVectors) {
5960 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
5962 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
5963 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5964 CFAOffset, FrameReg);
5982 if (
MI.isFullCopy()) {
5985 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
5989 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
5994 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
6022 if (
MI.isCopy() && Ops.
size() == 1 &&
6024 (Ops[0] == 0 || Ops[0] == 1)) {
6025 bool IsSpill = Ops[0] == 0;
6026 bool IsFill = !IsSpill;
6038 :
TRI.getMinimalPhysRegClass(Reg);
6044 "Mismatched register size in non subreg COPY");
6051 return &*--InsertPt;
6063 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
6066 "Unexpected subreg on physical register");
6068 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
6070 return &*--InsertPt;
6088 case AArch64::sub_32:
6089 FillRC = &AArch64::GPR32RegClass;
6092 FillRC = &AArch64::FPR32RegClass;
6095 FillRC = &AArch64::FPR64RegClass;
6101 TRI.getRegSizeInBits(*FillRC) &&
6102 "Mismatched regclass size on folded subreg COPY");
6121 bool *OutUseUnscaledOp,
6122 unsigned *OutUnscaledOp,
6123 int64_t *EmittableOffset) {
6125 if (EmittableOffset)
6126 *EmittableOffset = 0;
6127 if (OutUseUnscaledOp)
6128 *OutUseUnscaledOp =
false;
6134 switch (
MI.getOpcode()) {
6137 case AArch64::LD1Rv1d:
6138 case AArch64::LD1Rv2s:
6139 case AArch64::LD1Rv2d:
6140 case AArch64::LD1Rv4h:
6141 case AArch64::LD1Rv4s:
6142 case AArch64::LD1Rv8b:
6143 case AArch64::LD1Rv8h:
6144 case AArch64::LD1Rv16b:
6145 case AArch64::LD1Twov2d:
6146 case AArch64::LD1Threev2d:
6147 case AArch64::LD1Fourv2d:
6148 case AArch64::LD1Twov1d:
6149 case AArch64::LD1Threev1d:
6150 case AArch64::LD1Fourv1d:
6151 case AArch64::ST1Twov2d:
6152 case AArch64::ST1Threev2d:
6153 case AArch64::ST1Fourv2d:
6154 case AArch64::ST1Twov1d:
6155 case AArch64::ST1Threev1d:
6156 case AArch64::ST1Fourv1d:
6157 case AArch64::ST1i8:
6158 case AArch64::ST1i16:
6159 case AArch64::ST1i32:
6160 case AArch64::ST1i64:
6162 case AArch64::IRGstack:
6163 case AArch64::STGloop:
6164 case AArch64::STZGloop:
6169 TypeSize ScaleValue(0U,
false), Width(0U,
false);
6170 int64_t MinOff, MaxOff;
6176 bool IsMulVL = ScaleValue.isScalable();
6177 unsigned Scale = ScaleValue.getKnownMinValue();
6187 std::optional<unsigned> UnscaledOp =
6189 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
6190 if (useUnscaledOp &&
6195 Scale = ScaleValue.getKnownMinValue();
6196 assert(IsMulVL == ScaleValue.isScalable() &&
6197 "Unscaled opcode has different value for scalable");
6199 int64_t Remainder =
Offset % Scale;
6200 assert(!(Remainder && useUnscaledOp) &&
6201 "Cannot have remainder when using unscaled op");
6203 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
6204 int64_t NewOffset =
Offset / Scale;
6205 if (MinOff <= NewOffset && NewOffset <= MaxOff)
6208 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
6212 if (EmittableOffset)
6213 *EmittableOffset = NewOffset;
6214 if (OutUseUnscaledOp)
6215 *OutUseUnscaledOp = useUnscaledOp;
6216 if (OutUnscaledOp && UnscaledOp)
6217 *OutUnscaledOp = *UnscaledOp;
6230 unsigned Opcode =
MI.getOpcode();
6231 unsigned ImmIdx = FrameRegIdx + 1;
6233 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
6238 MI.eraseFromParent();
6244 unsigned UnscaledOp;
6247 &UnscaledOp, &NewOffset);
6251 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
6253 MI.setDesc(
TII->get(UnscaledOp));
6255 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
6278 case AArch64::ADDSWrr:
6279 case AArch64::ADDSWri:
6280 case AArch64::ADDSXrr:
6281 case AArch64::ADDSXri:
6282 case AArch64::SUBSWrr:
6283 case AArch64::SUBSXrr:
6285 case AArch64::SUBSWri:
6286 case AArch64::SUBSXri:
6297 case AArch64::ADDWrr:
6298 case AArch64::ADDWri:
6299 case AArch64::SUBWrr:
6300 case AArch64::ADDSWrr:
6301 case AArch64::ADDSWri:
6302 case AArch64::SUBSWrr:
6304 case AArch64::SUBWri:
6305 case AArch64::SUBSWri:
6316 case AArch64::ADDXrr:
6317 case AArch64::ADDXri:
6318 case AArch64::SUBXrr:
6319 case AArch64::ADDSXrr:
6320 case AArch64::ADDSXri:
6321 case AArch64::SUBSXrr:
6323 case AArch64::SUBXri:
6324 case AArch64::SUBSXri:
6325 case AArch64::ADDv8i8:
6326 case AArch64::ADDv16i8:
6327 case AArch64::ADDv4i16:
6328 case AArch64::ADDv8i16:
6329 case AArch64::ADDv2i32:
6330 case AArch64::ADDv4i32:
6331 case AArch64::SUBv8i8:
6332 case AArch64::SUBv16i8:
6333 case AArch64::SUBv4i16:
6334 case AArch64::SUBv8i16:
6335 case AArch64::SUBv2i32:
6336 case AArch64::SUBv4i32:
6349 case AArch64::FADDHrr:
6350 case AArch64::FADDSrr:
6351 case AArch64::FADDDrr:
6352 case AArch64::FADDv4f16:
6353 case AArch64::FADDv8f16:
6354 case AArch64::FADDv2f32:
6355 case AArch64::FADDv2f64:
6356 case AArch64::FADDv4f32:
6357 case AArch64::FSUBHrr:
6358 case AArch64::FSUBSrr:
6359 case AArch64::FSUBDrr:
6360 case AArch64::FSUBv4f16:
6361 case AArch64::FSUBv8f16:
6362 case AArch64::FSUBv2f32:
6363 case AArch64::FSUBv2f64:
6364 case AArch64::FSUBv4f32:
6368 return Options.UnsafeFPMath ||
6385 unsigned CombineOpc,
unsigned ZeroReg = 0,
6386 bool CheckZeroReg =
false) {
6393 if (!
MI ||
MI->getParent() != &
MBB || (
unsigned)
MI->getOpcode() != CombineOpc)
6396 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
6400 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
6401 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
6402 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
6404 if (
MI->getOperand(3).getReg() != ZeroReg)
6409 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
6418 unsigned MulOpc,
unsigned ZeroReg) {
6434 bool Invert)
const {
6440 case AArch64::FADDHrr:
6441 case AArch64::FADDSrr:
6442 case AArch64::FADDDrr:
6443 case AArch64::FMULHrr:
6444 case AArch64::FMULSrr:
6445 case AArch64::FMULDrr:
6446 case AArch64::FMULX16:
6447 case AArch64::FMULX32:
6448 case AArch64::FMULX64:
6450 case AArch64::FADDv4f16:
6451 case AArch64::FADDv8f16:
6452 case AArch64::FADDv2f32:
6453 case AArch64::FADDv4f32:
6454 case AArch64::FADDv2f64:
6455 case AArch64::FMULv4f16:
6456 case AArch64::FMULv8f16:
6457 case AArch64::FMULv2f32:
6458 case AArch64::FMULv4f32:
6459 case AArch64::FMULv2f64:
6460 case AArch64::FMULXv4f16:
6461 case AArch64::FMULXv8f16:
6462 case AArch64::FMULXv2f32:
6463 case AArch64::FMULXv4f32:
6464 case AArch64::FMULXv2f64:
6468 case AArch64::FADD_ZZZ_H:
6469 case AArch64::FADD_ZZZ_S:
6470 case AArch64::FADD_ZZZ_D:
6471 case AArch64::FMUL_ZZZ_H:
6472 case AArch64::FMUL_ZZZ_S:
6473 case AArch64::FMUL_ZZZ_D:
6485 case AArch64::ADDWrr:
6486 case AArch64::ADDXrr:
6487 case AArch64::ANDWrr:
6488 case AArch64::ANDXrr:
6489 case AArch64::ORRWrr:
6490 case AArch64::ORRXrr:
6491 case AArch64::EORWrr:
6492 case AArch64::EORXrr:
6493 case AArch64::EONWrr:
6494 case AArch64::EONXrr:
6498 case AArch64::ADDv8i8:
6499 case AArch64::ADDv16i8:
6500 case AArch64::ADDv4i16:
6501 case AArch64::ADDv8i16:
6502 case AArch64::ADDv2i32:
6503 case AArch64::ADDv4i32:
6504 case AArch64::ADDv1i64:
6505 case AArch64::ADDv2i64:
6506 case AArch64::MULv8i8:
6507 case AArch64::MULv16i8:
6508 case AArch64::MULv4i16:
6509 case AArch64::MULv8i16:
6510 case AArch64::MULv2i32:
6511 case AArch64::MULv4i32:
6512 case AArch64::ANDv8i8:
6513 case AArch64::ANDv16i8:
6514 case AArch64::ORRv8i8:
6515 case AArch64::ORRv16i8:
6516 case AArch64::EORv8i8:
6517 case AArch64::EORv16i8:
6519 case AArch64::ADD_ZZZ_B:
6520 case AArch64::ADD_ZZZ_H:
6521 case AArch64::ADD_ZZZ_S:
6522 case AArch64::ADD_ZZZ_D:
6523 case AArch64::MUL_ZZZ_B:
6524 case AArch64::MUL_ZZZ_H:
6525 case AArch64::MUL_ZZZ_S:
6526 case AArch64::MUL_ZZZ_D:
6527 case AArch64::AND_ZZZ:
6528 case AArch64::ORR_ZZZ:
6529 case AArch64::EOR_ZZZ:
6560 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6568 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6580 case AArch64::ADDWrr:
6582 "ADDWrr does not have register operands");
6583 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6584 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6586 case AArch64::ADDXrr:
6587 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6588 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6590 case AArch64::SUBWrr:
6591 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6592 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6594 case AArch64::SUBXrr:
6595 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6596 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6598 case AArch64::ADDWri:
6599 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6601 case AArch64::ADDXri:
6602 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6604 case AArch64::SUBWri:
6605 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6607 case AArch64::SUBXri:
6608 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6610 case AArch64::ADDv8i8:
6611 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6612 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6614 case AArch64::ADDv16i8:
6615 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6616 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6618 case AArch64::ADDv4i16:
6619 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6620 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6621 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6622 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6624 case AArch64::ADDv8i16:
6625 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6626 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6627 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6628 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6630 case AArch64::ADDv2i32:
6631 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6632 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6633 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6634 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6636 case AArch64::ADDv4i32:
6637 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6638 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6639 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6640 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6642 case AArch64::SUBv8i8:
6643 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6644 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6646 case AArch64::SUBv16i8:
6647 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6648 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6650 case AArch64::SUBv4i16:
6651 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6652 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
6653 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
6654 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
6656 case AArch64::SUBv8i16:
6657 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
6658 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
6659 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
6660 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
6662 case AArch64::SUBv2i32:
6663 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
6664 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
6665 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
6666 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
6668 case AArch64::SUBv4i32:
6669 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
6670 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
6671 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
6672 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
6689 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
6701 assert(
false &&
"Unsupported FP instruction in combiner\n");
6703 case AArch64::FADDHrr:
6705 "FADDHrr does not have register operands");
6707 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
6708 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
6710 case AArch64::FADDSrr:
6712 "FADDSrr does not have register operands");
6714 Found |=
Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
6715 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
6717 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
6718 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
6720 case AArch64::FADDDrr:
6721 Found |=
Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
6722 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
6724 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
6725 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
6727 case AArch64::FADDv4f16:
6728 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
6729 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
6731 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
6732 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
6734 case AArch64::FADDv8f16:
6735 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
6736 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
6738 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
6739 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
6741 case AArch64::FADDv2f32:
6742 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
6743 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
6745 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
6746 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
6748 case AArch64::FADDv2f64:
6749 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
6750 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
6752 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
6753 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
6755 case AArch64::FADDv4f32:
6756 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
6757 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
6759 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
6760 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
6762 case AArch64::FSUBHrr:
6763 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
6764 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
6765 Found |=
Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
6767 case AArch64::FSUBSrr:
6768 Found =
Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
6770 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
6771 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
6773 Found |=
Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
6775 case AArch64::FSUBDrr:
6776 Found =
Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
6778 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
6779 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
6781 Found |=
Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
6783 case AArch64::FSUBv4f16:
6784 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
6785 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
6787 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
6788 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
6790 case AArch64::FSUBv8f16:
6791 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
6792 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
6794 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
6795 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
6797 case AArch64::FSUBv2f32:
6798 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
6799 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
6801 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
6802 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
6804 case AArch64::FSUBv2f64:
6805 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
6806 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
6808 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
6809 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
6811 case AArch64::FSUBv4f32:
6812 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
6813 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
6815 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
6816 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
6827 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
6834 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
6835 MI->getOperand(1).getReg().isVirtual())
6836 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
6837 if (
MI &&
MI->getOpcode() == Opcode) {
6849 case AArch64::FMULv2f32:
6850 Found =
Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
6851 Found |=
Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
6853 case AArch64::FMULv2f64:
6854 Found =
Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
6855 Found |=
Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
6857 case AArch64::FMULv4f16:
6858 Found =
Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
6859 Found |=
Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
6861 case AArch64::FMULv4f32:
6862 Found =
Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
6863 Found |=
Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
6865 case AArch64::FMULv8f16:
6866 Found =
Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
6867 Found |=
Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
6880 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
6883 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
6884 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
6898 case AArch64::FNEGDr:
6900 case AArch64::FNEGSr:
7032 case AArch64::SUBWrr:
7033 case AArch64::SUBSWrr:
7034 case AArch64::SUBXrr:
7035 case AArch64::SUBSXrr:
7077 bool DoRegPressureReduce)
const {
7094 DoRegPressureReduce);
7123 const Register *ReplacedAddend =
nullptr) {
7124 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7126 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
7129 Register SrcReg0 = MUL->getOperand(1).getReg();
7130 bool Src0IsKill = MUL->getOperand(1).isKill();
7131 Register SrcReg1 = MUL->getOperand(2).getReg();
7132 bool Src1IsKill = MUL->getOperand(2).isKill();
7136 if (ReplacedAddend) {
7138 SrcReg2 = *ReplacedAddend;
7146 MRI.constrainRegClass(ResultReg, RC);
7148 MRI.constrainRegClass(SrcReg0, RC);
7150 MRI.constrainRegClass(SrcReg1, RC);
7152 MRI.constrainRegClass(SrcReg2, RC);
7155 if (kind == FMAInstKind::Default)
7160 else if (kind == FMAInstKind::Indexed)
7165 .
addImm(MUL->getOperand(3).getImm());
7166 else if (kind == FMAInstKind::Accumulator)
7172 assert(
false &&
"Invalid FMA instruction kind \n");
7186 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
7187 Opc = AArch64::FNMADDSrrr;
7188 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
7189 Opc = AArch64::FNMADDDrrr;
7201 MRI.constrainRegClass(ResultReg, RC);
7203 MRI.constrainRegClass(SrcReg0, RC);
7205 MRI.constrainRegClass(SrcReg1, RC);
7207 MRI.constrainRegClass(SrcReg2, RC);
7223 unsigned IdxDupOp,
unsigned MulOpc,
7225 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
7226 "Invalid index of FMUL operand");
7234 if (Dup->
getOpcode() == TargetOpcode::COPY)
7238 MRI.clearKillFlags(DupSrcReg);
7239 MRI.constrainRegClass(DupSrcReg, RC);
7243 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
7267 FMAInstKind::Accumulator);
7284 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7299 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
7301 FMAInstKind::Accumulator, &NewVR);
7313 FMAInstKind::Indexed);
7326 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
7329 FMAInstKind::Indexed, &NewVR);
7354 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
7356 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
7360 Register SrcReg0 = MUL->getOperand(1).getReg();
7361 bool Src0IsKill = MUL->getOperand(1).isKill();
7362 Register SrcReg1 = MUL->getOperand(2).getReg();
7363 bool Src1IsKill = MUL->getOperand(2).isKill();
7366 MRI.constrainRegClass(ResultReg, RC);
7368 MRI.constrainRegClass(SrcReg0, RC);
7370 MRI.constrainRegClass(SrcReg1, RC);
7372 MRI.constrainRegClass(VR, RC);
7394 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
7395 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
7409 if (Opcode == AArch64::SUBSWrr)
7410 Opcode = AArch64::SUBWrr;
7411 else if (Opcode == AArch64::SUBSXrr)
7412 Opcode = AArch64::SUBXrr;
7414 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
7415 "Unexpected instruction opcode.");
7432 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7459 DelInstrs, InstrIdxForVirtReg);
7465 InstrIdxForVirtReg);
7471 InstrIdxForVirtReg);
7480 Opc = AArch64::MADDWrrr;
7481 RC = &AArch64::GPR32RegClass;
7483 Opc = AArch64::MADDXrrr;
7484 RC = &AArch64::GPR64RegClass;
7495 Opc = AArch64::MADDWrrr;
7496 RC = &AArch64::GPR32RegClass;
7498 Opc = AArch64::MADDXrrr;
7499 RC = &AArch64::GPR64RegClass;
7511 unsigned BitSize, OrrOpc, ZeroReg;
7513 OrrOpc = AArch64::ORRWri;
7514 OrrRC = &AArch64::GPR32spRegClass;
7516 ZeroReg = AArch64::WZR;
7517 Opc = AArch64::MADDWrrr;
7518 RC = &AArch64::GPR32RegClass;
7520 OrrOpc = AArch64::ORRXri;
7521 OrrRC = &AArch64::GPR64spRegClass;
7523 ZeroReg = AArch64::XZR;
7524 Opc = AArch64::MADDXrrr;
7525 RC = &AArch64::GPR64RegClass;
7527 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7538 if (
Insn.size() != 1)
7540 auto MovI =
Insn.begin();
7543 if (MovI->Opcode == OrrOpc)
7549 assert((MovI->Opcode == AArch64::MOVNWi ||
7550 MovI->Opcode == AArch64::MOVZWi) &&
7553 assert((MovI->Opcode == AArch64::MOVNXi ||
7554 MovI->Opcode == AArch64::MOVZXi) &&
7561 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7562 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7573 unsigned SubOpc, ZeroReg;
7575 SubOpc = AArch64::SUBWrr;
7576 SubRC = &AArch64::GPR32spRegClass;
7577 ZeroReg = AArch64::WZR;
7578 Opc = AArch64::MADDWrrr;
7579 RC = &AArch64::GPR32RegClass;
7581 SubOpc = AArch64::SUBXrr;
7582 SubRC = &AArch64::GPR64spRegClass;
7583 ZeroReg = AArch64::XZR;
7584 Opc = AArch64::MADDXrrr;
7585 RC = &AArch64::GPR64RegClass;
7587 Register NewVR =
MRI.createVirtualRegister(SubRC);
7594 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7595 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7605 Opc = AArch64::MSUBWrrr;
7606 RC = &AArch64::GPR32RegClass;
7608 Opc = AArch64::MSUBXrrr;
7609 RC = &AArch64::GPR64RegClass;
7621 unsigned BitSize, OrrOpc, ZeroReg;
7623 OrrOpc = AArch64::ORRWri;
7624 OrrRC = &AArch64::GPR32spRegClass;
7626 ZeroReg = AArch64::WZR;
7627 Opc = AArch64::MADDWrrr;
7628 RC = &AArch64::GPR32RegClass;
7630 OrrOpc = AArch64::ORRXri;
7631 OrrRC = &AArch64::GPR64spRegClass;
7633 ZeroReg = AArch64::XZR;
7634 Opc = AArch64::MADDXrrr;
7635 RC = &AArch64::GPR64RegClass;
7637 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7647 if (
Insn.size() != 1)
7649 auto MovI =
Insn.begin();
7652 if (MovI->Opcode == OrrOpc)
7658 assert((MovI->Opcode == AArch64::MOVNWi ||
7659 MovI->Opcode == AArch64::MOVZWi) &&
7662 assert((MovI->Opcode == AArch64::MOVNXi ||
7663 MovI->Opcode == AArch64::MOVZXi) &&
7670 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7671 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7676 Opc = AArch64::MLAv8i8;
7677 RC = &AArch64::FPR64RegClass;
7681 Opc = AArch64::MLAv8i8;
7682 RC = &AArch64::FPR64RegClass;
7686 Opc = AArch64::MLAv16i8;
7687 RC = &AArch64::FPR128RegClass;
7691 Opc = AArch64::MLAv16i8;
7692 RC = &AArch64::FPR128RegClass;
7696 Opc = AArch64::MLAv4i16;
7697 RC = &AArch64::FPR64RegClass;
7701 Opc = AArch64::MLAv4i16;
7702 RC = &AArch64::FPR64RegClass;
7706 Opc = AArch64::MLAv8i16;
7707 RC = &AArch64::FPR128RegClass;
7711 Opc = AArch64::MLAv8i16;
7712 RC = &AArch64::FPR128RegClass;
7716 Opc = AArch64::MLAv2i32;
7717 RC = &AArch64::FPR64RegClass;
7721 Opc = AArch64::MLAv2i32;
7722 RC = &AArch64::FPR64RegClass;
7726 Opc = AArch64::MLAv4i32;
7727 RC = &AArch64::FPR128RegClass;
7731 Opc = AArch64::MLAv4i32;
7732 RC = &AArch64::FPR128RegClass;
7737 Opc = AArch64::MLAv8i8;
7738 RC = &AArch64::FPR64RegClass;
7740 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
7744 Opc = AArch64::MLSv8i8;
7745 RC = &AArch64::FPR64RegClass;
7749 Opc = AArch64::MLAv16i8;
7750 RC = &AArch64::FPR128RegClass;
7752 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
7756 Opc = AArch64::MLSv16i8;
7757 RC = &AArch64::FPR128RegClass;
7761 Opc = AArch64::MLAv4i16;
7762 RC = &AArch64::FPR64RegClass;
7764 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7768 Opc = AArch64::MLSv4i16;
7769 RC = &AArch64::FPR64RegClass;
7773 Opc = AArch64::MLAv8i16;
7774 RC = &AArch64::FPR128RegClass;
7776 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7780 Opc = AArch64::MLSv8i16;
7781 RC = &AArch64::FPR128RegClass;
7785 Opc = AArch64::MLAv2i32;
7786 RC = &AArch64::FPR64RegClass;
7788 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7792 Opc = AArch64::MLSv2i32;
7793 RC = &AArch64::FPR64RegClass;
7797 Opc = AArch64::MLAv4i32;
7798 RC = &AArch64::FPR128RegClass;
7800 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7804 Opc = AArch64::MLSv4i32;
7805 RC = &AArch64::FPR128RegClass;
7810 Opc = AArch64::MLAv4i16_indexed;
7811 RC = &AArch64::FPR64RegClass;
7815 Opc = AArch64::MLAv4i16_indexed;
7816 RC = &AArch64::FPR64RegClass;
7820 Opc = AArch64::MLAv8i16_indexed;
7821 RC = &AArch64::FPR128RegClass;
7825 Opc = AArch64::MLAv8i16_indexed;
7826 RC = &AArch64::FPR128RegClass;
7830 Opc = AArch64::MLAv2i32_indexed;
7831 RC = &AArch64::FPR64RegClass;
7835 Opc = AArch64::MLAv2i32_indexed;
7836 RC = &AArch64::FPR64RegClass;
7840 Opc = AArch64::MLAv4i32_indexed;
7841 RC = &AArch64::FPR128RegClass;
7845 Opc = AArch64::MLAv4i32_indexed;
7846 RC = &AArch64::FPR128RegClass;
7851 Opc = AArch64::MLAv4i16_indexed;
7852 RC = &AArch64::FPR64RegClass;
7854 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7858 Opc = AArch64::MLSv4i16_indexed;
7859 RC = &AArch64::FPR64RegClass;
7863 Opc = AArch64::MLAv8i16_indexed;
7864 RC = &AArch64::FPR128RegClass;
7866 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7870 Opc = AArch64::MLSv8i16_indexed;
7871 RC = &AArch64::FPR128RegClass;
7875 Opc = AArch64::MLAv2i32_indexed;
7876 RC = &AArch64::FPR64RegClass;
7878 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7882 Opc = AArch64::MLSv2i32_indexed;
7883 RC = &AArch64::FPR64RegClass;
7887 Opc = AArch64::MLAv4i32_indexed;
7888 RC = &AArch64::FPR128RegClass;
7890 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7894 Opc = AArch64::MLSv4i32_indexed;
7895 RC = &AArch64::FPR128RegClass;
7901 Opc = AArch64::FMADDHrrr;
7902 RC = &AArch64::FPR16RegClass;
7906 Opc = AArch64::FMADDSrrr;
7907 RC = &AArch64::FPR32RegClass;
7911 Opc = AArch64::FMADDDrrr;
7912 RC = &AArch64::FPR64RegClass;
7917 Opc = AArch64::FMADDHrrr;
7918 RC = &AArch64::FPR16RegClass;
7922 Opc = AArch64::FMADDSrrr;
7923 RC = &AArch64::FPR32RegClass;
7927 Opc = AArch64::FMADDDrrr;
7928 RC = &AArch64::FPR64RegClass;
7933 Opc = AArch64::FMLAv1i32_indexed;
7934 RC = &AArch64::FPR32RegClass;
7936 FMAInstKind::Indexed);
7939 Opc = AArch64::FMLAv1i32_indexed;
7940 RC = &AArch64::FPR32RegClass;
7942 FMAInstKind::Indexed);
7946 Opc = AArch64::FMLAv1i64_indexed;
7947 RC = &AArch64::FPR64RegClass;
7949 FMAInstKind::Indexed);
7952 Opc = AArch64::FMLAv1i64_indexed;
7953 RC = &AArch64::FPR64RegClass;
7955 FMAInstKind::Indexed);
7959 RC = &AArch64::FPR64RegClass;
7960 Opc = AArch64::FMLAv4i16_indexed;
7962 FMAInstKind::Indexed);
7965 RC = &AArch64::FPR64RegClass;
7966 Opc = AArch64::FMLAv4f16;
7968 FMAInstKind::Accumulator);
7971 RC = &AArch64::FPR64RegClass;
7972 Opc = AArch64::FMLAv4i16_indexed;
7974 FMAInstKind::Indexed);
7977 RC = &AArch64::FPR64RegClass;
7978 Opc = AArch64::FMLAv4f16;
7980 FMAInstKind::Accumulator);
7985 RC = &AArch64::FPR64RegClass;
7987 Opc = AArch64::FMLAv2i32_indexed;
7989 FMAInstKind::Indexed);
7991 Opc = AArch64::FMLAv2f32;
7993 FMAInstKind::Accumulator);
7998 RC = &AArch64::FPR64RegClass;
8000 Opc = AArch64::FMLAv2i32_indexed;
8002 FMAInstKind::Indexed);
8004 Opc = AArch64::FMLAv2f32;
8006 FMAInstKind::Accumulator);
8011 RC = &AArch64::FPR128RegClass;
8012 Opc = AArch64::FMLAv8i16_indexed;
8014 FMAInstKind::Indexed);
8017 RC = &AArch64::FPR128RegClass;
8018 Opc = AArch64::FMLAv8f16;
8020 FMAInstKind::Accumulator);
8023 RC = &AArch64::FPR128RegClass;
8024 Opc = AArch64::FMLAv8i16_indexed;
8026 FMAInstKind::Indexed);
8029 RC = &AArch64::FPR128RegClass;
8030 Opc = AArch64::FMLAv8f16;
8032 FMAInstKind::Accumulator);
8037 RC = &AArch64::FPR128RegClass;
8039 Opc = AArch64::FMLAv2i64_indexed;
8041 FMAInstKind::Indexed);
8043 Opc = AArch64::FMLAv2f64;
8045 FMAInstKind::Accumulator);
8050 RC = &AArch64::FPR128RegClass;
8052 Opc = AArch64::FMLAv2i64_indexed;
8054 FMAInstKind::Indexed);
8056 Opc = AArch64::FMLAv2f64;
8058 FMAInstKind::Accumulator);
8064 RC = &AArch64::FPR128RegClass;
8066 Opc = AArch64::FMLAv4i32_indexed;
8068 FMAInstKind::Indexed);
8070 Opc = AArch64::FMLAv4f32;
8072 FMAInstKind::Accumulator);
8078 RC = &AArch64::FPR128RegClass;
8080 Opc = AArch64::FMLAv4i32_indexed;
8082 FMAInstKind::Indexed);
8084 Opc = AArch64::FMLAv4f32;
8086 FMAInstKind::Accumulator);
8091 Opc = AArch64::FNMSUBHrrr;
8092 RC = &AArch64::FPR16RegClass;
8096 Opc = AArch64::FNMSUBSrrr;
8097 RC = &AArch64::FPR32RegClass;
8101 Opc = AArch64::FNMSUBDrrr;
8102 RC = &AArch64::FPR64RegClass;
8107 Opc = AArch64::FNMADDHrrr;
8108 RC = &AArch64::FPR16RegClass;
8112 Opc = AArch64::FNMADDSrrr;
8113 RC = &AArch64::FPR32RegClass;
8117 Opc = AArch64::FNMADDDrrr;
8118 RC = &AArch64::FPR64RegClass;
8123 Opc = AArch64::FMSUBHrrr;
8124 RC = &AArch64::FPR16RegClass;
8128 Opc = AArch64::FMSUBSrrr;
8129 RC = &AArch64::FPR32RegClass;
8133 Opc = AArch64::FMSUBDrrr;
8134 RC = &AArch64::FPR64RegClass;
8139 Opc = AArch64::FMLSv1i32_indexed;
8140 RC = &AArch64::FPR32RegClass;
8142 FMAInstKind::Indexed);
8146 Opc = AArch64::FMLSv1i64_indexed;
8147 RC = &AArch64::FPR64RegClass;
8149 FMAInstKind::Indexed);
8154 RC = &AArch64::FPR64RegClass;
8160 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8162 Opc = AArch64::FMLAv4f16;
8164 FMAInstKind::Accumulator, &NewVR);
8166 Opc = AArch64::FMLAv4i16_indexed;
8168 FMAInstKind::Indexed, &NewVR);
8173 RC = &AArch64::FPR64RegClass;
8174 Opc = AArch64::FMLSv4f16;
8176 FMAInstKind::Accumulator);
8179 RC = &AArch64::FPR64RegClass;
8180 Opc = AArch64::FMLSv4i16_indexed;
8182 FMAInstKind::Indexed);
8187 RC = &AArch64::FPR64RegClass;
8189 Opc = AArch64::FMLSv2i32_indexed;
8191 FMAInstKind::Indexed);
8193 Opc = AArch64::FMLSv2f32;
8195 FMAInstKind::Accumulator);
8201 RC = &AArch64::FPR128RegClass;
8207 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8209 Opc = AArch64::FMLAv8f16;
8211 FMAInstKind::Accumulator, &NewVR);
8213 Opc = AArch64::FMLAv8i16_indexed;
8215 FMAInstKind::Indexed, &NewVR);
8220 RC = &AArch64::FPR128RegClass;
8221 Opc = AArch64::FMLSv8f16;
8223 FMAInstKind::Accumulator);
8226 RC = &AArch64::FPR128RegClass;
8227 Opc = AArch64::FMLSv8i16_indexed;
8229 FMAInstKind::Indexed);
8234 RC = &AArch64::FPR128RegClass;
8236 Opc = AArch64::FMLSv2i64_indexed;
8238 FMAInstKind::Indexed);
8240 Opc = AArch64::FMLSv2f64;
8242 FMAInstKind::Accumulator);
8248 RC = &AArch64::FPR128RegClass;
8250 Opc = AArch64::FMLSv4i32_indexed;
8252 FMAInstKind::Indexed);
8254 Opc = AArch64::FMLSv4f32;
8256 FMAInstKind::Accumulator);
8261 RC = &AArch64::FPR64RegClass;
8267 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8269 Opc = AArch64::FMLAv2i32_indexed;
8271 FMAInstKind::Indexed, &NewVR);
8273 Opc = AArch64::FMLAv2f32;
8275 FMAInstKind::Accumulator, &NewVR);
8281 RC = &AArch64::FPR128RegClass;
8287 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8289 Opc = AArch64::FMLAv4i32_indexed;
8291 FMAInstKind::Indexed, &NewVR);
8293 Opc = AArch64::FMLAv4f32;
8295 FMAInstKind::Accumulator, &NewVR);
8301 RC = &AArch64::FPR128RegClass;
8307 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
8309 Opc = AArch64::FMLAv2i64_indexed;
8311 FMAInstKind::Indexed, &NewVR);
8313 Opc = AArch64::FMLAv2f64;
8315 FMAInstKind::Accumulator, &NewVR);
8325 &AArch64::FPR128RegClass,
MRI);
8334 &AArch64::FPR128RegClass,
MRI);
8343 &AArch64::FPR128_loRegClass,
MRI);
8352 &AArch64::FPR128RegClass,
MRI);
8361 &AArch64::FPR128_loRegClass,
MRI);
8380 for (
auto *
MI : InsInstrs)
8381 MI->setFlags(Flags);
8422 bool IsNegativeBranch =
false;
8423 bool IsTestAndBranch =
false;
8424 unsigned TargetBBInMI = 0;
8425 switch (
MI.getOpcode()) {
8434 case AArch64::CBNZW:
8435 case AArch64::CBNZX:
8437 IsNegativeBranch =
true;
8442 IsTestAndBranch =
true;
8444 case AArch64::TBNZW:
8445 case AArch64::TBNZX:
8447 IsNegativeBranch =
true;
8448 IsTestAndBranch =
true;
8454 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
8458 assert(
MI.getParent() &&
"Incomplete machine instruciton\n");
8471 if (!
MRI->hasOneNonDBGUse(CopyVReg))
8473 if (!
MRI->hasOneDef(CopyVReg))
8482 case AArch64::ANDWri:
8483 case AArch64::ANDXri: {
8484 if (IsTestAndBranch)
8488 if (!
MRI->hasOneNonDBGUse(VReg))
8502 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
8508 unsigned Opc = (Imm < 32)
8509 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
8510 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
8523 if (!Is32Bit && Imm < 32)
8525 MI.eraseFromParent();
8529 case AArch64::CSINCWr:
8530 case AArch64::CSINCXr: {
8550 if (IsNegativeBranch)
8553 MI.eraseFromParent();
8559std::pair<unsigned, unsigned>
8562 return std::make_pair(TF & Mask, TF & ~Mask);
8567 using namespace AArch64II;
8569 static const std::pair<unsigned, const char *> TargetFlags[] = {
8570 {MO_PAGE,
"aarch64-page"}, {MO_PAGEOFF,
"aarch64-pageoff"},
8571 {MO_G3,
"aarch64-g3"}, {MO_G2,
"aarch64-g2"},
8572 {MO_G1,
"aarch64-g1"}, {MO_G0,
"aarch64-g0"},
8573 {MO_HI12,
"aarch64-hi12"}};
8579 using namespace AArch64II;
8581 static const std::pair<unsigned, const char *> TargetFlags[] = {
8582 {MO_COFFSTUB,
"aarch64-coffstub"},
8583 {MO_GOT,
"aarch64-got"},
8584 {MO_NC,
"aarch64-nc"},
8585 {MO_S,
"aarch64-s"},
8586 {MO_TLS,
"aarch64-tls"},
8587 {MO_DLLIMPORT,
"aarch64-dllimport"},
8588 {MO_PREL,
"aarch64-prel"},
8589 {MO_TAGGED,
"aarch64-tagged"},
8590 {MO_ARM64EC_CALLMANGLE,
"aarch64-arm64ec-callmangle"},
8597 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8705 for (
unsigned Reg : AArch64::GPR64RegClass) {
8707 Reg != AArch64::LR &&
8708 Reg != AArch64::X16 &&
8709 Reg != AArch64::X17 &&
8710 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
8711 C.isAvailableInsideSeq(
Reg,
TRI))
8742 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
8745std::optional<std::unique_ptr<outliner::OutlinedFunction>>
8748 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
8749 unsigned MinRepeats)
const {
8750 unsigned SequenceSize = 0;
8751 for (
auto &
MI : RepeatedSequenceLocs[0])
8754 unsigned NumBytesToCreateFrame = 0;
8764 if (std::adjacent_find(
8765 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
8769 if (outliningCandidatesSigningScopeConsensus(a, b) &&
8770 outliningCandidatesSigningKeyConsensus(a, b) &&
8771 outliningCandidatesV8_3OpsConsensus(a, b)) {
8775 }) != RepeatedSequenceLocs.end()) {
8776 return std::nullopt;
8793 unsigned NumBytesToCheckLRInTCEpilogue = 0;
8794 if (RepeatedSequenceLocs[0]
8796 ->getInfo<AArch64FunctionInfo>()
8797 ->shouldSignReturnAddress(
true)) {
8799 NumBytesToCreateFrame += 8;
8802 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
8803 *RepeatedSequenceLocs[0].getMF());
8804 NumBytesToCheckLRInTCEpilogue =
8808 if (isTailCallReturnInst(RepeatedSequenceLocs[0].back()))
8809 SequenceSize += NumBytesToCheckLRInTCEpilogue;
8817 for (
auto &
MI :
C) {
8818 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
8819 switch (
MI.getOpcode()) {
8820 case AArch64::ADDXri:
8821 case AArch64::ADDWri:
8822 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8824 "Expected operand to be immediate");
8826 "Expected operand to be a register");
8830 if (
MI.getOperand(1).getReg() == AArch64::SP)
8831 SPValue +=
MI.getOperand(2).getImm();
8835 case AArch64::SUBXri:
8836 case AArch64::SUBWri:
8837 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8839 "Expected operand to be immediate");
8841 "Expected operand to be a register");
8845 if (
MI.getOperand(1).getReg() == AArch64::SP)
8846 SPValue -=
MI.getOperand(2).getImm();
8863 if (RepeatedSequenceLocs.size() < MinRepeats)
8864 return std::nullopt;
8868 unsigned FlagsSetInAll = 0xF;
8872 FlagsSetInAll &=
C.Flags;
8874 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
8877 auto SetCandidateCallInfo =
8878 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
8880 C.setCallInfo(CallID, NumBytesForCall);
8884 NumBytesToCreateFrame += 4;
8887 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
8892 unsigned CFICount = 0;
8893 for (
auto &
I : RepeatedSequenceLocs[0]) {
8894 if (
I.isCFIInstruction())
8904 std::vector<MCCFIInstruction> CFIInstructions =
8905 C.getMF()->getFrameInstructions();
8907 if (CFICount > 0 && CFICount != CFIInstructions.size())
8908 return std::nullopt;
8916 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
8917 !
MI.readsRegister(AArch64::SP, &
TRI))
8923 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
8928 if (
MI.mayLoadOrStore()) {
8931 bool OffsetIsScalable;
8935 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
8936 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
8940 if (OffsetIsScalable)
8948 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
8949 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
8952 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
8953 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
8968 bool AllStackInstrsSafe =
8973 if (RepeatedSequenceLocs[0].back().isTerminator()) {
8975 NumBytesToCreateFrame = 0;
8976 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
8980 else if (LastInstrOpcode == AArch64::BL ||
8981 ((LastInstrOpcode == AArch64::BLR ||
8982 LastInstrOpcode == AArch64::BLRNoIP) &&
8986 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
8994 unsigned NumBytesNoStackCalls = 0;
8995 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
9000 (
C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
9001 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
9010 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
9013 if (LRAvailable && !IsNoReturn) {
9014 NumBytesNoStackCalls += 4;
9016 CandidatesWithoutStackFixups.push_back(
C);
9021 else if (findRegisterToSaveLRTo(
C)) {
9022 NumBytesNoStackCalls += 12;
9024 CandidatesWithoutStackFixups.push_back(
C);
9029 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
9030 NumBytesNoStackCalls += 12;
9032 CandidatesWithoutStackFixups.push_back(
C);
9038 NumBytesNoStackCalls += SequenceSize;
9045 if (!AllStackInstrsSafe ||
9046 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
9047 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
9049 if (RepeatedSequenceLocs.size() < MinRepeats)
9050 return std::nullopt;
9099 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
9103 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
9104 !findRegisterToSaveLRTo(
C));
9110 if (RepeatedSequenceLocs.size() < MinRepeats)
9111 return std::nullopt;
9116 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
9120 bool ModStackToSaveLR =
false;
9123 ModStackToSaveLR =
true;
9132 ModStackToSaveLR =
true;
9134 if (ModStackToSaveLR) {
9136 if (!AllStackInstrsSafe)
9137 return std::nullopt;
9140 NumBytesToCreateFrame += 8;
9147 return std::nullopt;
9149 return std::make_unique<outliner::OutlinedFunction>(
9150 RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID);
9154 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
9158 const auto &CFn = Candidates.front().getMF()->getFunction();
9160 if (CFn.hasFnAttribute(
"ptrauth-returns"))
9161 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-returns"));
9162 if (CFn.hasFnAttribute(
"ptrauth-auth-traps"))
9163 F.addFnAttr(CFn.getFnAttribute(
"ptrauth-auth-traps"));
9166 if (CFn.hasFnAttribute(
"sign-return-address"))
9167 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
9168 if (CFn.hasFnAttribute(
"sign-return-address-key"))
9169 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
9171 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
9179 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
9193 if (!AFI || AFI->
hasRedZone().value_or(
true))
9213 unsigned &Flags)
const {
9215 "Must track liveness!");
9217 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
9232 auto AreAllUnsafeRegsDead = [&LRU]() {
9249 bool LRAvailableEverywhere =
true;
9254 if (
MI.isCall() && !
MI.isTerminator())
9255 Flags |= MachineOutlinerMBBFlags::HasCalls;
9260 auto CreateNewRangeStartingAt =
9261 [&RangeBegin, &RangeEnd,
9263 RangeBegin = NewBegin;
9264 RangeEnd = std::next(RangeBegin);
9267 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
9272 Ranges.push_back(std::make_pair(RangeBegin, RangeEnd));
9280 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
9284 UpdateWholeMBBFlags(*FirstPossibleEndPt);
9285 if (AreAllUnsafeRegsDead())
9292 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
9298 UpdateWholeMBBFlags(
MI);
9299 if (!AreAllUnsafeRegsDead()) {
9300 SaveRangeIfNonEmpty();
9301 CreateNewRangeStartingAt(
MI.getIterator());
9304 LRAvailableEverywhere &= LRU.
available(AArch64::LR);
9305 RangeBegin =
MI.getIterator();
9310 if (AreAllUnsafeRegsDead())
9311 SaveRangeIfNonEmpty();
9316 std::reverse(Ranges.begin(), Ranges.end());
9319 if (!LRAvailableEverywhere)
9320 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
9327 unsigned Flags)
const {
9335 switch (
MI.getOpcode()) {
9337 case AArch64::PACIASP:
9338 case AArch64::PACIBSP:
9339 case AArch64::PACIASPPC:
9340 case AArch64::PACIBSPPC:
9341 case AArch64::AUTIASP:
9342 case AArch64::AUTIBSP:
9343 case AArch64::AUTIASPPCi:
9344 case AArch64::AUTIASPPCr:
9345 case AArch64::AUTIBSPPCi:
9346 case AArch64::AUTIBSPPCr:
9347 case AArch64::RETAA:
9348 case AArch64::RETAB:
9349 case AArch64::RETAASPPCi:
9350 case AArch64::RETAASPPCr:
9351 case AArch64::RETABSPPCi:
9352 case AArch64::RETABSPPCr:
9353 case AArch64::EMITBKEY:
9354 case AArch64::PAUTH_PROLOGUE:
9355 case AArch64::PAUTH_EPILOGUE:
9369 if (
MI.isCFIInstruction())
9373 if (
MI.isTerminator())
9382 assert(!MOP.isCFIIndex());
9385 if (MOP.isReg() && !MOP.isImplicit() &&
9386 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
9393 if (
MI.getOpcode() == AArch64::ADRP)
9414 if (MOP.isGlobal()) {
9415 Callee = dyn_cast<Function>(MOP.getGlobal());
9422 if (Callee && Callee->getName() ==
"\01_mcount")
9430 if (
MI.getOpcode() == AArch64::BLR ||
9431 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
9435 return UnknownCallOutlineType;
9443 return UnknownCallOutlineType;
9451 return UnknownCallOutlineType;
9476 bool OffsetIsScalable;
9479 if (!
MI.mayLoadOrStore() ||
9482 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
9487 int64_t Dummy1, Dummy2;
9490 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
9492 assert(Scale != 0 &&
"Unexpected opcode!");
9493 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
9498 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
9499 StackOffsetOperand.
setImm(NewImm);
9505 bool ShouldSignReturnAddr) {
9506 if (!ShouldSignReturnAddr)
9512 TII->get(AArch64::PAUTH_EPILOGUE))
9528 unsigned TailOpcode;
9529 if (Call->getOpcode() == AArch64::BL) {
9530 TailOpcode = AArch64::TCRETURNdi;
9532 assert(Call->getOpcode() == AArch64::BLR ||
9533 Call->getOpcode() == AArch64::BLRNoIP);
9534 TailOpcode = AArch64::TCRETURNriALL;
9537 .
add(Call->getOperand(0))
9540 Call->eraseFromParent();
9545 bool IsLeafFunction =
true;
9549 return MI.isCall() && !
MI.isReturn();
9559 "Can only fix up stack references once");
9560 fixupPostOutline(
MBB);
9562 IsLeafFunction =
false;
9573 Et = std::prev(
MBB.
end());
9586 unsigned DwarfReg =
MRI->getDwarfRegNum(AArch64::LR,
true);
9589 int64_t StackPosEntry =
9642 fixupPostOutline(
MBB);
9653 .addGlobalAddress(M.getNamedValue(MF.
getName()))
9663 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9678 Register Reg = findRegisterToSaveLRTo(
C);
9679 assert(Reg &&
"No callee-saved register available?");
9713 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9729 bool AllowSideEffects)
const {
9734 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
9747 assert(STI.hasNEON() &&
"Expected to have NEON.");
9748 Register Reg64 =
TRI.getSubReg(Reg, AArch64::dsub);
9753std::optional<DestSourcePair>
9758 if (((
MI.getOpcode() == AArch64::ORRWrs &&
9759 MI.getOperand(1).getReg() == AArch64::WZR &&
9760 MI.getOperand(3).getImm() == 0x0) ||
9761 (
MI.getOpcode() == AArch64::ORRWrr &&
9762 MI.getOperand(1).getReg() == AArch64::WZR)) &&
9764 (!
MI.getOperand(0).getReg().isVirtual() ||
9765 MI.getOperand(0).getSubReg() == 0) &&
9766 (!
MI.getOperand(0).getReg().isPhysical() ||
9767 MI.findRegisterDefOperandIdx(
MI.getOperand(0).getReg() - AArch64::W0 +
9772 if (
MI.getOpcode() == AArch64::ORRXrs &&
9773 MI.getOperand(1).getReg() == AArch64::XZR &&
9774 MI.getOperand(3).getImm() == 0x0)
9777 return std::nullopt;
9780std::optional<DestSourcePair>
9782 if ((
MI.getOpcode() == AArch64::ORRWrs &&
9783 MI.getOperand(1).getReg() == AArch64::WZR &&
9784 MI.getOperand(3).getImm() == 0x0) ||
9785 (
MI.getOpcode() == AArch64::ORRWrr &&
9786 MI.getOperand(1).getReg() == AArch64::WZR))
9788 return std::nullopt;
9791std::optional<RegImmPair>
9800 return std::nullopt;
9802 switch (
MI.getOpcode()) {
9804 return std::nullopt;
9805 case AArch64::SUBWri:
9806 case AArch64::SUBXri:
9807 case AArch64::SUBSWri:
9808 case AArch64::SUBSXri:
9811 case AArch64::ADDSWri:
9812 case AArch64::ADDSXri:
9813 case AArch64::ADDWri:
9814 case AArch64::ADDXri: {
9816 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
9817 !
MI.getOperand(2).isImm())
9818 return std::nullopt;
9819 int Shift =
MI.getOperand(3).getImm();
9820 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
9821 Offset = Sign * (
MI.getOperand(2).getImm() << Shift);
9830static std::optional<ParamLoadedValue>
9834 auto DestSrc =
TII->isCopyLikeInstr(
MI);
9836 return std::nullopt;
9838 Register DestReg = DestSrc->Destination->getReg();
9839 Register SrcReg = DestSrc->Source->getReg();
9844 if (DestReg == DescribedReg)
9848 if (
MI.getOpcode() == AArch64::ORRWrs &&
9849 TRI->isSuperRegister(DestReg, DescribedReg))
9853 if (
MI.getOpcode() == AArch64::ORRXrs &&
9854 TRI->isSubRegister(DestReg, DescribedReg)) {
9855 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
9859 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
9860 "Unhandled ORR[XW]rs copy case");
9862 return std::nullopt;
9881 return MI.getOpcode() == AArch64::INLINEASM_BR;
9899 switch (
MI.getOpcode()) {
9900 case TargetOpcode::G_BRJT:
9901 case AArch64::JumpTableDest32:
9902 case AArch64::JumpTableDest16:
9903 case AArch64::JumpTableDest8:
9914std::optional<ParamLoadedValue>
9919 switch (
MI.getOpcode()) {
9920 case AArch64::MOVZWi:
9921 case AArch64::MOVZXi: {
9924 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
9925 return std::nullopt;
9927 if (!
MI.getOperand(1).isImm())
9928 return std::nullopt;
9929 int64_t Immediate =
MI.getOperand(1).getImm();
9930 int Shift =
MI.getOperand(2).getImm();
9934 case AArch64::ORRWrs:
9935 case AArch64::ORRXrs:
9945 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
9946 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
9949 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
9953 if (!
MRI.hasOneNonDBGUse(DefReg))
9958 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
9959 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
9980 unsigned Scale)
const {
9991 unsigned Shift =
Log2_64(NumBytes);
9992 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
10000 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
10005 return AArch64::BLRNoIP;
10007 return AArch64::BLR;
10012 Register TargetReg,
bool FrameSetup)
const {
10013 assert(TargetReg != AArch64::SP &&
"New top of stack cannot aleady be in SP");
10025 MF.
insert(MBBInsertPoint, LoopTestMBB);
10028 MF.
insert(MBBInsertPoint, LoopBodyMBB);
10030 MF.
insert(MBBInsertPoint, ExitMBB);
10040 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
10048 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::Bcc))
10054 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
10067 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
10092 return ExitMBB->
begin();
10109 unsigned CompCounterOprNum;
10113 unsigned UpdateCounterOprNum;
10117 bool IsUpdatePriorComp;
10129 TII(MF->getSubtarget().getInstrInfo()),
10130 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
10131 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
10132 CompCounterOprNum(CompCounterOprNum), Update(Update),
10133 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
10134 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
10136 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
10142 std::optional<bool> createTripCountGreaterCondition(
10152 void createRemainingIterationsGreaterCondition(
10158 void adjustTripCount(
int TripCountAdjust)
override {}
10160 bool isMVEExpanderSupported()
override {
return true; }
10178 Result =
MRI.createVirtualRegister(
10181 }
else if (
I == ReplaceOprNum) {
10182 MRI.constrainRegClass(
10192void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
10208 assert(CondBranch->getOpcode() == AArch64::Bcc);
10212 if (CondBranch->getOperand(1).getMBB() == LoopBB)
10219 auto AccumulateCond = [&](
Register CurCond,
10221 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
10230 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
10234 for (
int I = 0;
I <= TC; ++
I) {
10240 AccCond = AccumulateCond(AccCond,
CC);
10244 if (Update != Comp && IsUpdatePriorComp) {
10246 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
10247 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
10251 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
10253 }
else if (Update != Comp) {
10258 Counter = NextCounter;
10262 if (LastStage0Insts.
empty()) {
10266 if (IsUpdatePriorComp)
10271 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
10274 for (
int I = 0;
I <= TC; ++
I) {
10278 AccCond = AccumulateCond(AccCond,
CC);
10279 if (
I != TC && Update != Comp)
10282 Counter = NextCounter;
10298 assert(Phi.getNumOperands() == 5);
10299 if (Phi.getOperand(2).getMBB() ==
MBB) {
10300 RegMBB = Phi.getOperand(1).getReg();
10301 RegOther = Phi.getOperand(3).getReg();
10303 assert(Phi.getOperand(4).getMBB() ==
MBB);
10304 RegMBB = Phi.getOperand(3).getReg();
10305 RegOther = Phi.getOperand(1).getReg();
10310 if (!Reg.isVirtual())
10313 return MRI.getVRegDef(Reg)->getParent() != BB;
10319 unsigned &UpdateCounterOprNum,
Register &InitReg,
10320 bool &IsUpdatePriorComp) {
10334 if (!Reg.isVirtual())
10337 UpdateInst =
nullptr;
10338 UpdateCounterOprNum = 0;
10340 IsUpdatePriorComp =
true;
10344 if (Def->getParent() != LoopBB)
10346 if (Def->isCopy()) {
10348 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
10350 CurReg = Def->getOperand(1).getReg();
10351 }
else if (Def->isPHI()) {
10355 IsUpdatePriorComp =
false;
10360 switch (Def->getOpcode()) {
10361 case AArch64::ADDSXri:
10362 case AArch64::ADDSWri:
10363 case AArch64::SUBSXri:
10364 case AArch64::SUBSWri:
10365 case AArch64::ADDXri:
10366 case AArch64::ADDWri:
10367 case AArch64::SUBXri:
10368 case AArch64::SUBWri:
10370 UpdateCounterOprNum = 1;
10372 case AArch64::ADDSXrr:
10373 case AArch64::ADDSWrr:
10374 case AArch64::SUBSXrr:
10375 case AArch64::SUBSWrr:
10376 case AArch64::ADDXrr:
10377 case AArch64::ADDWrr:
10378 case AArch64::SUBXrr:
10379 case AArch64::SUBWrr:
10382 UpdateCounterOprNum = 1;
10384 UpdateCounterOprNum = 2;
10391 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
10406std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
10417 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
10428 if (
TBB == LoopBB && FBB == LoopBB)
10432 if (
TBB != LoopBB && FBB ==
nullptr)
10435 assert((
TBB == LoopBB || FBB == LoopBB) &&
10436 "The Loop must be a single-basic-block loop");
10441 if (CondBranch->
getOpcode() != AArch64::Bcc)
10449 unsigned CompCounterOprNum = 0;
10451 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
10455 switch (
MI.getOpcode()) {
10456 case AArch64::SUBSXri:
10457 case AArch64::SUBSWri:
10458 case AArch64::ADDSXri:
10459 case AArch64::ADDSWri:
10461 CompCounterOprNum = 1;
10463 case AArch64::ADDSWrr:
10464 case AArch64::ADDSXrr:
10465 case AArch64::SUBSWrr:
10466 case AArch64::SUBSXrr:
10477 if (CompCounterOprNum == 0) {
10479 CompCounterOprNum = 2;
10481 CompCounterOprNum = 1;
10493 bool IsUpdatePriorComp;
10494 unsigned UpdateCounterOprNum;
10496 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
10499 return std::make_unique<AArch64PipelinerLoopInfo>(
10500 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
10511 TypeSize Scale(0U,
false), Width(0U,
false);
10512 int64_t MinOffset, MaxOffset;
10513 if (
getMemOpInfo(
MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
10515 if (
MI.getOperand(ImmIdx).isImm() && !
MI.getOperand(ImmIdx - 1).isFI()) {
10516 int64_t Imm =
MI.getOperand(ImmIdx).getImm();
10517 if (Imm < MinOffset || Imm > MaxOffset) {
10518 ErrInfo =
"Unexpected immediate on load/store instruction";
10526#define GET_INSTRINFO_HELPERS
10527#define GET_INSTRMAP_INFO
10528#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, MCRegister Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static bool isPostIndexLdStOpcode(unsigned Opcode)
Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
static unsigned getBranchDisplacementBits(unsigned Opc)
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallVector class.
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setOutliningStyle(const std::string &Style)
const SetOfInstructions & getLOHRelated() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors, int64_t &NumDataVectors)
Returns the offset in parts to which this frame offset can be decomposed for the purpose of describin...
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
uint64_t getElementSizeForOpcode(unsigned Opc) const
Returns the vector element size (B, H, S or D) of an SVE opcode.
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
bool isWhileOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE WHILE## instruction.
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
SmallVector< std::pair< MachineBasicBlock::iterator, MachineBasicBlock::iterator > > getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool useMachineCombiner() const override
AArch64 supports MachineCombiner.
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const override
static bool isFalkorShiftExtFast(const MachineInstr &MI)
Returns true if the instruction has a shift by immediate that can be executed in one cycle less.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
bool expandPostRAPseudo(MachineInstr &MI) const override
unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
bool isFunctionSafeToSplit(const MachineFunction &MF) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
Return true when Inst is associative and commutative so that it can be reassociated.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset, unsigned Scale) const
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableBitmaskMachineOperandTargetFlags() const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isPTestLikeOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE instruction that sets the condition codes as if it's results...
void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const override
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isTargetILP32() const
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
AArch64PAuth::AuthCheckMethod getAuthenticatedLRCheckMethod(const MachineFunction &MF) const
Choose a method of checking LR before performing a tail call.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MBBSectionID getSectionID() const
Returns the section ID of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
Primary interface to the complete machine description for the target machine.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool succeeded(LogicalResult Result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
static bool isIndirectBranchOpcode(int Opc)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
unsigned getUndefRegState(bool B)
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
DWARFExpression::Operation Op
static bool isUncondBranchOpcode(int Opc)
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
static const MachineMemOperand::Flags MOStridedAccess
@ Default
The result values are uniform if and only if all operands are uniform.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
static const MBBSectionID ColdSectionID
MachineJumpTableEntry - One jump table in the jump table info.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.