63#define GET_INSTRINFO_CTOR_DTOR
64#include "AArch64GenInstrInfo.inc"
68 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
72 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
76 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
80 cl::desc(
"Restrict range of B instructions (DEBUG)"));
85 RI(STI.getTargetTriple()), Subtarget(STI) {}
96 auto Op =
MI.getOpcode();
97 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
98 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
102 if (
MI.isMetaInstruction())
107 unsigned NumBytes = 0;
113 switch (
Desc.getOpcode()) {
116 return Desc.getSize();
123 case TargetOpcode::STACKMAP:
126 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
128 case TargetOpcode::PATCHPOINT:
131 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
133 case TargetOpcode::STATEPOINT:
135 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
140 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
145 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
147 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
148 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
152 case TargetOpcode::PATCHABLE_EVENT_CALL:
158 NumBytes =
MI.getOperand(1).getImm();
160 case TargetOpcode::BUNDLE:
161 NumBytes = getInstBundleLength(
MI);
168unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
172 while (++
I != E &&
I->isInsideBundle()) {
173 assert(!
I->isBundle() &&
"No nested bundle!");
232 int64_t BrOffset)
const {
234 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
235 "over conditional branch expansion");
236 return isIntN(Bits, BrOffset / 4);
241 switch (
MI.getOpcode()) {
245 return MI.getOperand(0).getMBB();
250 return MI.getOperand(2).getMBB();
256 return MI.getOperand(1).getMBB();
266 assert(RS &&
"RegScavenger required for long branching");
268 "new block should be inserted for expanding unconditional branch");
271 "restore block should be inserted for restoring clobbered registers");
276 if (!isInt<33>(BrOffset))
278 "Branch offsets outside of the signed 33-bit range not supported");
292 constexpr Register Reg = AArch64::X16;
294 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
302 if (Scavenged != AArch64::NoRegister &&
304 buildIndirectBranch(Scavenged, NewDestBB);
314 "Unable to insert indirect branch inside function that has red zone");
337 bool AllowModify)
const {
344 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
345 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
349 if (!isUnpredicatedTerminator(*
I))
356 unsigned LastOpc = LastInst->
getOpcode();
357 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
372 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
379 LastInst = SecondLastInst;
381 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
386 SecondLastInst = &*
I;
387 SecondLastOpc = SecondLastInst->
getOpcode();
398 LastInst = SecondLastInst;
400 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
402 "unreachable unconditional branches removed above");
411 SecondLastInst = &*
I;
412 SecondLastOpc = SecondLastInst->
getOpcode();
416 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
432 I->eraseFromParent();
441 I->eraseFromParent();
450 MachineBranchPredicate &MBP,
451 bool AllowModify)
const {
461 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
462 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
466 if (!isUnpredicatedTerminator(*
I))
471 unsigned LastOpc = LastInst->
getOpcode();
486 assert(MBP.TrueDest &&
"expected!");
489 MBP.ConditionDef =
nullptr;
490 MBP.SingleUseCondition =
false;
494 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
495 : MachineBranchPredicate::PRED_EQ;
501 if (
Cond[0].getImm() != -1) {
507 switch (
Cond[1].getImm()) {
511 Cond[1].setImm(AArch64::CBNZW);
514 Cond[1].setImm(AArch64::CBZW);
517 Cond[1].setImm(AArch64::CBNZX);
520 Cond[1].setImm(AArch64::CBZX);
523 Cond[1].setImm(AArch64::TBNZW);
526 Cond[1].setImm(AArch64::TBZW);
529 Cond[1].setImm(AArch64::TBNZX);
532 Cond[1].setImm(AArch64::TBZX);
541 int *BytesRemoved)
const {
551 I->eraseFromParent();
568 I->eraseFromParent();
575void AArch64InstrInfo::instantiateCondBranch(
578 if (
Cond[0].getImm() != -1) {
596 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
635 unsigned *NewVReg =
nullptr) {
640 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
643 unsigned SrcOpNum = 0;
645 case AArch64::ADDSXri:
646 case AArch64::ADDSWri:
653 case AArch64::ADDXri:
654 case AArch64::ADDWri:
660 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
663 case AArch64::ORNXrr:
664 case AArch64::ORNWrr: {
667 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
670 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
674 case AArch64::SUBSXrr:
675 case AArch64::SUBSWrr:
682 case AArch64::SUBXrr:
683 case AArch64::SUBWrr: {
686 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
689 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
695 assert(Opc && SrcOpNum &&
"Missing parameters");
707 int &FalseCycles)
const {
711 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
718 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
722 unsigned ExtraCondLat =
Cond.size() != 1;
726 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
727 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
729 CondCycles = 1 + ExtraCondLat;
730 TrueCycles = FalseCycles = 1;
740 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
741 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
742 CondCycles = 5 + ExtraCondLat;
743 TrueCycles = FalseCycles = 2;
760 switch (
Cond.size()) {
769 switch (
Cond[1].getImm()) {
792 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
798 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
808 switch (
Cond[1].getImm()) {
821 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
837 bool TryFold =
false;
838 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
839 RC = &AArch64::GPR64RegClass;
840 Opc = AArch64::CSELXr;
842 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
843 RC = &AArch64::GPR32RegClass;
844 Opc = AArch64::CSELWr;
846 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
847 RC = &AArch64::FPR64RegClass;
848 Opc = AArch64::FCSELDrrr;
849 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
850 RC = &AArch64::FPR32RegClass;
851 Opc = AArch64::FCSELSrrr;
853 assert(RC &&
"Unsupported regclass");
857 unsigned NewVReg = 0;
872 MRI.clearKillFlags(NewVReg);
877 MRI.constrainRegClass(TrueReg, RC);
878 MRI.constrainRegClass(FalseReg, RC);
893 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
898 return Is.
size() <= 2;
904 if (Subtarget.hasExynosCheapAsMoveHandling()) {
905 if (isExynosCheapAsMove(
MI))
907 return MI.isAsCheapAsAMove();
910 switch (
MI.getOpcode()) {
912 return MI.isAsCheapAsAMove();
914 case AArch64::ADDWrs:
915 case AArch64::ADDXrs:
916 case AArch64::SUBWrs:
917 case AArch64::SUBXrs:
918 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
923 case AArch64::MOVi32imm:
925 case AArch64::MOVi64imm:
931 switch (
MI.getOpcode()) {
935 case AArch64::ADDWrs:
936 case AArch64::ADDXrs:
937 case AArch64::ADDSWrs:
938 case AArch64::ADDSXrs: {
939 unsigned Imm =
MI.getOperand(3).getImm();
946 case AArch64::ADDWrx:
947 case AArch64::ADDXrx:
948 case AArch64::ADDXrx64:
949 case AArch64::ADDSWrx:
950 case AArch64::ADDSXrx:
951 case AArch64::ADDSXrx64: {
952 unsigned Imm =
MI.getOperand(3).getImm();
964 case AArch64::SUBWrs:
965 case AArch64::SUBSWrs: {
966 unsigned Imm =
MI.getOperand(3).getImm();
968 return ShiftVal == 0 ||
972 case AArch64::SUBXrs:
973 case AArch64::SUBSXrs: {
974 unsigned Imm =
MI.getOperand(3).getImm();
976 return ShiftVal == 0 ||
980 case AArch64::SUBWrx:
981 case AArch64::SUBXrx:
982 case AArch64::SUBXrx64:
983 case AArch64::SUBSWrx:
984 case AArch64::SUBSXrx:
985 case AArch64::SUBSXrx64: {
986 unsigned Imm =
MI.getOperand(3).getImm();
998 case AArch64::LDRBBroW:
999 case AArch64::LDRBBroX:
1000 case AArch64::LDRBroW:
1001 case AArch64::LDRBroX:
1002 case AArch64::LDRDroW:
1003 case AArch64::LDRDroX:
1004 case AArch64::LDRHHroW:
1005 case AArch64::LDRHHroX:
1006 case AArch64::LDRHroW:
1007 case AArch64::LDRHroX:
1008 case AArch64::LDRQroW:
1009 case AArch64::LDRQroX:
1010 case AArch64::LDRSBWroW:
1011 case AArch64::LDRSBWroX:
1012 case AArch64::LDRSBXroW:
1013 case AArch64::LDRSBXroX:
1014 case AArch64::LDRSHWroW:
1015 case AArch64::LDRSHWroX:
1016 case AArch64::LDRSHXroW:
1017 case AArch64::LDRSHXroX:
1018 case AArch64::LDRSWroW:
1019 case AArch64::LDRSWroX:
1020 case AArch64::LDRSroW:
1021 case AArch64::LDRSroX:
1022 case AArch64::LDRWroW:
1023 case AArch64::LDRWroX:
1024 case AArch64::LDRXroW:
1025 case AArch64::LDRXroX:
1026 case AArch64::PRFMroW:
1027 case AArch64::PRFMroX:
1028 case AArch64::STRBBroW:
1029 case AArch64::STRBBroX:
1030 case AArch64::STRBroW:
1031 case AArch64::STRBroX:
1032 case AArch64::STRDroW:
1033 case AArch64::STRDroX:
1034 case AArch64::STRHHroW:
1035 case AArch64::STRHHroX:
1036 case AArch64::STRHroW:
1037 case AArch64::STRHroX:
1038 case AArch64::STRQroW:
1039 case AArch64::STRQroX:
1040 case AArch64::STRSroW:
1041 case AArch64::STRSroX:
1042 case AArch64::STRWroW:
1043 case AArch64::STRWroX:
1044 case AArch64::STRXroW:
1045 case AArch64::STRXroX: {
1046 unsigned IsSigned =
MI.getOperand(3).getImm();
1053 unsigned Opc =
MI.getOpcode();
1057 case AArch64::SEH_StackAlloc:
1058 case AArch64::SEH_SaveFPLR:
1059 case AArch64::SEH_SaveFPLR_X:
1060 case AArch64::SEH_SaveReg:
1061 case AArch64::SEH_SaveReg_X:
1062 case AArch64::SEH_SaveRegP:
1063 case AArch64::SEH_SaveRegP_X:
1064 case AArch64::SEH_SaveFReg:
1065 case AArch64::SEH_SaveFReg_X:
1066 case AArch64::SEH_SaveFRegP:
1067 case AArch64::SEH_SaveFRegP_X:
1068 case AArch64::SEH_SetFP:
1069 case AArch64::SEH_AddFP:
1070 case AArch64::SEH_Nop:
1071 case AArch64::SEH_PrologEnd:
1072 case AArch64::SEH_EpilogStart:
1073 case AArch64::SEH_EpilogEnd:
1074 case AArch64::SEH_PACSignLR:
1075 case AArch64::SEH_SaveAnyRegQP:
1076 case AArch64::SEH_SaveAnyRegQPX:
1083 unsigned &SubIdx)
const {
1084 switch (
MI.getOpcode()) {
1087 case AArch64::SBFMXri:
1088 case AArch64::UBFMXri:
1091 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1094 SrcReg =
MI.getOperand(1).getReg();
1095 DstReg =
MI.getOperand(0).getReg();
1096 SubIdx = AArch64::sub_32;
1105 int64_t OffsetA = 0, OffsetB = 0;
1106 TypeSize WidthA(0,
false), WidthB(0,
false);
1107 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1128 OffsetAIsScalable == OffsetBIsScalable) {
1129 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1130 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1131 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1132 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1150 switch (
MI.getOpcode()) {
1153 if (
MI.getOperand(0).getImm() == 0x14)
1160 case AArch64::MSRpstatesvcrImm1:
1167 auto Next = std::next(
MI.getIterator());
1168 return Next !=
MBB->
end() && Next->isCFIInstruction();
1175 Register &SrcReg2, int64_t &CmpMask,
1176 int64_t &CmpValue)
const {
1179 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1180 if (!
MI.getOperand(1).isReg())
1183 switch (
MI.getOpcode()) {
1186 case AArch64::PTEST_PP:
1187 case AArch64::PTEST_PP_ANY:
1188 SrcReg =
MI.getOperand(0).getReg();
1189 SrcReg2 =
MI.getOperand(1).getReg();
1194 case AArch64::SUBSWrr:
1195 case AArch64::SUBSWrs:
1196 case AArch64::SUBSWrx:
1197 case AArch64::SUBSXrr:
1198 case AArch64::SUBSXrs:
1199 case AArch64::SUBSXrx:
1200 case AArch64::ADDSWrr:
1201 case AArch64::ADDSWrs:
1202 case AArch64::ADDSWrx:
1203 case AArch64::ADDSXrr:
1204 case AArch64::ADDSXrs:
1205 case AArch64::ADDSXrx:
1207 SrcReg =
MI.getOperand(1).getReg();
1208 SrcReg2 =
MI.getOperand(2).getReg();
1212 case AArch64::SUBSWri:
1213 case AArch64::ADDSWri:
1214 case AArch64::SUBSXri:
1215 case AArch64::ADDSXri:
1216 SrcReg =
MI.getOperand(1).getReg();
1219 CmpValue =
MI.getOperand(2).getImm();
1221 case AArch64::ANDSWri:
1222 case AArch64::ANDSXri:
1225 SrcReg =
MI.getOperand(1).getReg();
1229 MI.getOperand(2).getImm(),
1230 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1239 assert(
MBB &&
"Can't get MachineBasicBlock here");
1241 assert(MF &&
"Can't get MachineFunction here");
1246 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1250 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1253 if (!OpRegCstraints)
1261 "Operand has register constraints without being a register!");
1264 if (Reg.isPhysical()) {
1265 if (!OpRegCstraints->
contains(Reg))
1268 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1281 bool MIDefinesZeroReg =
false;
1282 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1283 MI.definesRegister(AArch64::XZR,
nullptr))
1284 MIDefinesZeroReg =
true;
1286 switch (
MI.getOpcode()) {
1288 return MI.getOpcode();
1289 case AArch64::ADDSWrr:
1290 return AArch64::ADDWrr;
1291 case AArch64::ADDSWri:
1292 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1293 case AArch64::ADDSWrs:
1294 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1295 case AArch64::ADDSWrx:
1296 return AArch64::ADDWrx;
1297 case AArch64::ADDSXrr:
1298 return AArch64::ADDXrr;
1299 case AArch64::ADDSXri:
1300 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1301 case AArch64::ADDSXrs:
1302 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1303 case AArch64::ADDSXrx:
1304 return AArch64::ADDXrx;
1305 case AArch64::SUBSWrr:
1306 return AArch64::SUBWrr;
1307 case AArch64::SUBSWri:
1308 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1309 case AArch64::SUBSWrs:
1310 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1311 case AArch64::SUBSWrx:
1312 return AArch64::SUBWrx;
1313 case AArch64::SUBSXrr:
1314 return AArch64::SUBXrr;
1315 case AArch64::SUBSXri:
1316 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1317 case AArch64::SUBSXrs:
1318 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1319 case AArch64::SUBSXrx:
1320 return AArch64::SUBXrx;
1335 if (To == To->getParent()->begin())
1340 if (To->getParent() !=
From->getParent())
1352 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1353 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1359std::optional<unsigned>
1363 unsigned MaskOpcode =
Mask->getOpcode();
1364 unsigned PredOpcode = Pred->
getOpcode();
1368 if (PredIsWhileLike) {
1372 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1386 if (PredIsPTestLike) {
1391 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1401 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1430 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1438 switch (PredOpcode) {
1439 case AArch64::AND_PPzPP:
1440 case AArch64::BIC_PPzPP:
1441 case AArch64::EOR_PPzPP:
1442 case AArch64::NAND_PPzPP:
1443 case AArch64::NOR_PPzPP:
1444 case AArch64::ORN_PPzPP:
1445 case AArch64::ORR_PPzPP:
1446 case AArch64::BRKA_PPzP:
1447 case AArch64::BRKPA_PPzPP:
1448 case AArch64::BRKB_PPzP:
1449 case AArch64::BRKPB_PPzPP:
1450 case AArch64::RDFFR_PPz: {
1454 if (Mask != PredMask)
1458 case AArch64::BRKN_PPzP: {
1462 if ((MaskOpcode != AArch64::PTRUE_B) ||
1463 (
Mask->getOperand(1).getImm() != 31))
1467 case AArch64::PTRUE_B:
1480bool AArch64InstrInfo::optimizePTestInstr(
1481 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1483 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1484 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1485 unsigned PredOpcode = Pred->
getOpcode();
1486 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1502 if (*NewOp != PredOpcode) {
1513 for (; i !=
e; ++i) {
1544 if (DeadNZCVIdx != -1) {
1563 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1564 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1565 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1574 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1576 return (CmpValue == 0 || CmpValue == 1) &&
1577 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1585 switch (Instr.getOpcode()) {
1587 return AArch64::INSTRUCTION_LIST_END;
1589 case AArch64::ADDSWrr:
1590 case AArch64::ADDSWri:
1591 case AArch64::ADDSXrr:
1592 case AArch64::ADDSXri:
1593 case AArch64::SUBSWrr:
1594 case AArch64::SUBSWri:
1595 case AArch64::SUBSXrr:
1596 case AArch64::SUBSXri:
1597 return Instr.getOpcode();
1599 case AArch64::ADDWrr:
1600 return AArch64::ADDSWrr;
1601 case AArch64::ADDWri:
1602 return AArch64::ADDSWri;
1603 case AArch64::ADDXrr:
1604 return AArch64::ADDSXrr;
1605 case AArch64::ADDXri:
1606 return AArch64::ADDSXri;
1607 case AArch64::ADCWr:
1608 return AArch64::ADCSWr;
1609 case AArch64::ADCXr:
1610 return AArch64::ADCSXr;
1611 case AArch64::SUBWrr:
1612 return AArch64::SUBSWrr;
1613 case AArch64::SUBWri:
1614 return AArch64::SUBSWri;
1615 case AArch64::SUBXrr:
1616 return AArch64::SUBSXrr;
1617 case AArch64::SUBXri:
1618 return AArch64::SUBSXri;
1619 case AArch64::SBCWr:
1620 return AArch64::SBCSWr;
1621 case AArch64::SBCXr:
1622 return AArch64::SBCSXr;
1623 case AArch64::ANDWri:
1624 return AArch64::ANDSWri;
1625 case AArch64::ANDXri:
1626 return AArch64::ANDSXri;
1633 if (BB->isLiveIn(AArch64::NZCV))
1642 switch (Instr.getOpcode()) {
1646 case AArch64::Bcc: {
1647 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1652 case AArch64::CSINVWr:
1653 case AArch64::CSINVXr:
1654 case AArch64::CSINCWr:
1655 case AArch64::CSINCXr:
1656 case AArch64::CSELWr:
1657 case AArch64::CSELXr:
1658 case AArch64::CSNEGWr:
1659 case AArch64::CSNEGXr:
1660 case AArch64::FCSELSrrr:
1661 case AArch64::FCSELDrrr: {
1662 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1675 Instr.getOperand(CCIdx).getImm())
1728std::optional<UsedNZCV>
1733 if (
MI.getParent() != CmpParent)
1734 return std::nullopt;
1737 return std::nullopt;
1742 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1745 return std::nullopt;
1750 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1753 return NZCVUsedAfterCmp;
1757 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1761 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1783 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1789 "Caller guarantees that CmpInstr compares with constant 0");
1792 if (!NZVCUsed || NZVCUsed->C)
1814bool AArch64InstrInfo::substituteCmpToZero(
1825 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1832 MI->setDesc(
get(NewOpc));
1837 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1849 assert((CmpValue == 0 || CmpValue == 1) &&
1850 "Only comparisons to 0 or 1 considered for removal!");
1853 unsigned MIOpc =
MI.getOpcode();
1854 if (MIOpc == AArch64::CSINCWr) {
1855 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1856 MI.getOperand(2).getReg() != AArch64::WZR)
1858 }
else if (MIOpc == AArch64::CSINCXr) {
1859 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1860 MI.getOperand(2).getReg() != AArch64::XZR)
1870 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1874 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1876 if (CmpValue && !IsSubsRegImm)
1878 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1883 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1886 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1890 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1893 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1894 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1897 if (MIUsedNZCV.
N && !CmpValue)
1939bool AArch64InstrInfo::removeCmpToZeroOrOne(
1947 bool IsInvertCC =
false;
1957 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1968 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1969 MI.getOpcode() != AArch64::CATCHRET)
1977 if (
MI.getOpcode() == AArch64::CATCHRET) {
1986 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1988 FirstEpilogSEH = std::next(FirstEpilogSEH);
2002 if (M.getStackProtectorGuard() ==
"sysreg") {
2012 int Offset = M.getStackProtectorGuardOffset();
2064 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2073 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2115 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2138 switch (
MI.getOpcode()) {
2141 case AArch64::MOVZWi:
2142 case AArch64::MOVZXi:
2143 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2144 assert(
MI.getDesc().getNumOperands() == 3 &&
2145 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2149 case AArch64::ANDWri:
2150 return MI.getOperand(1).getReg() == AArch64::WZR;
2151 case AArch64::ANDXri:
2152 return MI.getOperand(1).getReg() == AArch64::XZR;
2153 case TargetOpcode::COPY:
2154 return MI.getOperand(1).getReg() == AArch64::WZR;
2162 switch (
MI.getOpcode()) {
2165 case TargetOpcode::COPY: {
2168 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2169 AArch64::GPR64RegClass.
contains(DstReg));
2171 case AArch64::ORRXrs:
2172 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2173 assert(
MI.getDesc().getNumOperands() == 4 &&
2174 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2178 case AArch64::ADDXri:
2179 if (
MI.getOperand(2).getImm() == 0) {
2180 assert(
MI.getDesc().getNumOperands() == 4 &&
2181 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2192 switch (
MI.getOpcode()) {
2195 case TargetOpcode::COPY: {
2197 return AArch64::FPR128RegClass.contains(DstReg);
2199 case AArch64::ORRv16i8:
2200 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2201 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2202 "invalid ORRv16i8 operands");
2211 int &FrameIndex)
const {
2212 switch (
MI.getOpcode()) {
2215 case AArch64::LDRWui:
2216 case AArch64::LDRXui:
2217 case AArch64::LDRBui:
2218 case AArch64::LDRHui:
2219 case AArch64::LDRSui:
2220 case AArch64::LDRDui:
2221 case AArch64::LDRQui:
2222 case AArch64::LDR_PXI:
2223 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2224 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2225 FrameIndex =
MI.getOperand(1).getIndex();
2226 return MI.getOperand(0).getReg();
2235 int &FrameIndex)
const {
2236 switch (
MI.getOpcode()) {
2239 case AArch64::STRWui:
2240 case AArch64::STRXui:
2241 case AArch64::STRBui:
2242 case AArch64::STRHui:
2243 case AArch64::STRSui:
2244 case AArch64::STRDui:
2245 case AArch64::STRQui:
2246 case AArch64::STR_PXI:
2247 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2248 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2249 FrameIndex =
MI.getOperand(1).getIndex();
2250 return MI.getOperand(0).getReg();
2260 return MMO->getFlags() & MOSuppressPair;
2266 if (
MI.memoperands_empty())
2274 return MMO->getFlags() & MOStridedAccess;
2282 case AArch64::STURSi:
2283 case AArch64::STRSpre:
2284 case AArch64::STURDi:
2285 case AArch64::STRDpre:
2286 case AArch64::STURQi:
2287 case AArch64::STRQpre:
2288 case AArch64::STURBBi:
2289 case AArch64::STURHHi:
2290 case AArch64::STURWi:
2291 case AArch64::STRWpre:
2292 case AArch64::STURXi:
2293 case AArch64::STRXpre:
2294 case AArch64::LDURSi:
2295 case AArch64::LDRSpre:
2296 case AArch64::LDURDi:
2297 case AArch64::LDRDpre:
2298 case AArch64::LDURQi:
2299 case AArch64::LDRQpre:
2300 case AArch64::LDURWi:
2301 case AArch64::LDRWpre:
2302 case AArch64::LDURXi:
2303 case AArch64::LDRXpre:
2304 case AArch64::LDRSWpre:
2305 case AArch64::LDURSWi:
2306 case AArch64::LDURHHi:
2307 case AArch64::LDURBBi:
2308 case AArch64::LDURSBWi:
2309 case AArch64::LDURSHWi:
2317 case AArch64::PRFMui:
return AArch64::PRFUMi;
2318 case AArch64::LDRXui:
return AArch64::LDURXi;
2319 case AArch64::LDRWui:
return AArch64::LDURWi;
2320 case AArch64::LDRBui:
return AArch64::LDURBi;
2321 case AArch64::LDRHui:
return AArch64::LDURHi;
2322 case AArch64::LDRSui:
return AArch64::LDURSi;
2323 case AArch64::LDRDui:
return AArch64::LDURDi;
2324 case AArch64::LDRQui:
return AArch64::LDURQi;
2325 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2326 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2327 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2328 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2329 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2330 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2331 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2332 case AArch64::STRXui:
return AArch64::STURXi;
2333 case AArch64::STRWui:
return AArch64::STURWi;
2334 case AArch64::STRBui:
return AArch64::STURBi;
2335 case AArch64::STRHui:
return AArch64::STURHi;
2336 case AArch64::STRSui:
return AArch64::STURSi;
2337 case AArch64::STRDui:
return AArch64::STURDi;
2338 case AArch64::STRQui:
return AArch64::STURQi;
2339 case AArch64::STRBBui:
return AArch64::STURBBi;
2340 case AArch64::STRHHui:
return AArch64::STURHHi;
2348 case AArch64::LDPXi:
2349 case AArch64::LDPDi:
2350 case AArch64::STPXi:
2351 case AArch64::STPDi:
2352 case AArch64::LDNPXi:
2353 case AArch64::LDNPDi:
2354 case AArch64::STNPXi:
2355 case AArch64::STNPDi:
2356 case AArch64::LDPQi:
2357 case AArch64::STPQi:
2358 case AArch64::LDNPQi:
2359 case AArch64::STNPQi:
2360 case AArch64::LDPWi:
2361 case AArch64::LDPSi:
2362 case AArch64::STPWi:
2363 case AArch64::STPSi:
2364 case AArch64::LDNPWi:
2365 case AArch64::LDNPSi:
2366 case AArch64::STNPWi:
2367 case AArch64::STNPSi:
2369 case AArch64::STGPi:
2371 case AArch64::LD1B_IMM:
2372 case AArch64::LD1B_H_IMM:
2373 case AArch64::LD1B_S_IMM:
2374 case AArch64::LD1B_D_IMM:
2375 case AArch64::LD1SB_H_IMM:
2376 case AArch64::LD1SB_S_IMM:
2377 case AArch64::LD1SB_D_IMM:
2378 case AArch64::LD1H_IMM:
2379 case AArch64::LD1H_S_IMM:
2380 case AArch64::LD1H_D_IMM:
2381 case AArch64::LD1SH_S_IMM:
2382 case AArch64::LD1SH_D_IMM:
2383 case AArch64::LD1W_IMM:
2384 case AArch64::LD1W_D_IMM:
2385 case AArch64::LD1SW_D_IMM:
2386 case AArch64::LD1D_IMM:
2388 case AArch64::LD2B_IMM:
2389 case AArch64::LD2H_IMM:
2390 case AArch64::LD2W_IMM:
2391 case AArch64::LD2D_IMM:
2392 case AArch64::LD3B_IMM:
2393 case AArch64::LD3H_IMM:
2394 case AArch64::LD3W_IMM:
2395 case AArch64::LD3D_IMM:
2396 case AArch64::LD4B_IMM:
2397 case AArch64::LD4H_IMM:
2398 case AArch64::LD4W_IMM:
2399 case AArch64::LD4D_IMM:
2401 case AArch64::ST1B_IMM:
2402 case AArch64::ST1B_H_IMM:
2403 case AArch64::ST1B_S_IMM:
2404 case AArch64::ST1B_D_IMM:
2405 case AArch64::ST1H_IMM:
2406 case AArch64::ST1H_S_IMM:
2407 case AArch64::ST1H_D_IMM:
2408 case AArch64::ST1W_IMM:
2409 case AArch64::ST1W_D_IMM:
2410 case AArch64::ST1D_IMM:
2412 case AArch64::ST2B_IMM:
2413 case AArch64::ST2H_IMM:
2414 case AArch64::ST2W_IMM:
2415 case AArch64::ST2D_IMM:
2416 case AArch64::ST3B_IMM:
2417 case AArch64::ST3H_IMM:
2418 case AArch64::ST3W_IMM:
2419 case AArch64::ST3D_IMM:
2420 case AArch64::ST4B_IMM:
2421 case AArch64::ST4H_IMM:
2422 case AArch64::ST4W_IMM:
2423 case AArch64::ST4D_IMM:
2425 case AArch64::LD1RB_IMM:
2426 case AArch64::LD1RB_H_IMM:
2427 case AArch64::LD1RB_S_IMM:
2428 case AArch64::LD1RB_D_IMM:
2429 case AArch64::LD1RSB_H_IMM:
2430 case AArch64::LD1RSB_S_IMM:
2431 case AArch64::LD1RSB_D_IMM:
2432 case AArch64::LD1RH_IMM:
2433 case AArch64::LD1RH_S_IMM:
2434 case AArch64::LD1RH_D_IMM:
2435 case AArch64::LD1RSH_S_IMM:
2436 case AArch64::LD1RSH_D_IMM:
2437 case AArch64::LD1RW_IMM:
2438 case AArch64::LD1RW_D_IMM:
2439 case AArch64::LD1RSW_IMM:
2440 case AArch64::LD1RD_IMM:
2442 case AArch64::LDNT1B_ZRI:
2443 case AArch64::LDNT1H_ZRI:
2444 case AArch64::LDNT1W_ZRI:
2445 case AArch64::LDNT1D_ZRI:
2446 case AArch64::STNT1B_ZRI:
2447 case AArch64::STNT1H_ZRI:
2448 case AArch64::STNT1W_ZRI:
2449 case AArch64::STNT1D_ZRI:
2451 case AArch64::LDNF1B_IMM:
2452 case AArch64::LDNF1B_H_IMM:
2453 case AArch64::LDNF1B_S_IMM:
2454 case AArch64::LDNF1B_D_IMM:
2455 case AArch64::LDNF1SB_H_IMM:
2456 case AArch64::LDNF1SB_S_IMM:
2457 case AArch64::LDNF1SB_D_IMM:
2458 case AArch64::LDNF1H_IMM:
2459 case AArch64::LDNF1H_S_IMM:
2460 case AArch64::LDNF1H_D_IMM:
2461 case AArch64::LDNF1SH_S_IMM:
2462 case AArch64::LDNF1SH_D_IMM:
2463 case AArch64::LDNF1W_IMM:
2464 case AArch64::LDNF1W_D_IMM:
2465 case AArch64::LDNF1SW_D_IMM:
2466 case AArch64::LDNF1D_IMM:
2470 case AArch64::LDR_PXI:
2471 case AArch64::STR_PXI:
2477 switch (
MI.getOpcode()) {
2481 case AArch64::STRSui:
2482 case AArch64::STRDui:
2483 case AArch64::STRQui:
2484 case AArch64::STRXui:
2485 case AArch64::STRWui:
2486 case AArch64::LDRSui:
2487 case AArch64::LDRDui:
2488 case AArch64::LDRQui:
2489 case AArch64::LDRXui:
2490 case AArch64::LDRWui:
2491 case AArch64::LDRSWui:
2493 case AArch64::STURSi:
2494 case AArch64::STRSpre:
2495 case AArch64::STURDi:
2496 case AArch64::STRDpre:
2497 case AArch64::STURQi:
2498 case AArch64::STRQpre:
2499 case AArch64::STURWi:
2500 case AArch64::STRWpre:
2501 case AArch64::STURXi:
2502 case AArch64::STRXpre:
2503 case AArch64::LDURSi:
2504 case AArch64::LDRSpre:
2505 case AArch64::LDURDi:
2506 case AArch64::LDRDpre:
2507 case AArch64::LDURQi:
2508 case AArch64::LDRQpre:
2509 case AArch64::LDURWi:
2510 case AArch64::LDRWpre:
2511 case AArch64::LDURXi:
2512 case AArch64::LDRXpre:
2513 case AArch64::LDURSWi:
2514 case AArch64::LDRSWpre:
2520 switch (
MI.getOpcode()) {
2523 "Unexpected instruction - was a new tail call opcode introduced?");
2525 case AArch64::TCRETURNdi:
2526 case AArch64::TCRETURNri:
2527 case AArch64::TCRETURNrix16x17:
2528 case AArch64::TCRETURNrix17:
2529 case AArch64::TCRETURNrinotx16:
2530 case AArch64::TCRETURNriALL:
2531 case AArch64::AUTH_TCRETURN:
2532 case AArch64::AUTH_TCRETURN_BTI:
2542 case AArch64::ADDWri:
2543 return AArch64::ADDSWri;
2544 case AArch64::ADDWrr:
2545 return AArch64::ADDSWrr;
2546 case AArch64::ADDWrs:
2547 return AArch64::ADDSWrs;
2548 case AArch64::ADDWrx:
2549 return AArch64::ADDSWrx;
2550 case AArch64::ANDWri:
2551 return AArch64::ANDSWri;
2552 case AArch64::ANDWrr:
2553 return AArch64::ANDSWrr;
2554 case AArch64::ANDWrs:
2555 return AArch64::ANDSWrs;
2556 case AArch64::BICWrr:
2557 return AArch64::BICSWrr;
2558 case AArch64::BICWrs:
2559 return AArch64::BICSWrs;
2560 case AArch64::SUBWri:
2561 return AArch64::SUBSWri;
2562 case AArch64::SUBWrr:
2563 return AArch64::SUBSWrr;
2564 case AArch64::SUBWrs:
2565 return AArch64::SUBSWrs;
2566 case AArch64::SUBWrx:
2567 return AArch64::SUBSWrx;
2569 case AArch64::ADDXri:
2570 return AArch64::ADDSXri;
2571 case AArch64::ADDXrr:
2572 return AArch64::ADDSXrr;
2573 case AArch64::ADDXrs:
2574 return AArch64::ADDSXrs;
2575 case AArch64::ADDXrx:
2576 return AArch64::ADDSXrx;
2577 case AArch64::ANDXri:
2578 return AArch64::ANDSXri;
2579 case AArch64::ANDXrr:
2580 return AArch64::ANDSXrr;
2581 case AArch64::ANDXrs:
2582 return AArch64::ANDSXrs;
2583 case AArch64::BICXrr:
2584 return AArch64::BICSXrr;
2585 case AArch64::BICXrs:
2586 return AArch64::BICSXrs;
2587 case AArch64::SUBXri:
2588 return AArch64::SUBSXri;
2589 case AArch64::SUBXrr:
2590 return AArch64::SUBSXrr;
2591 case AArch64::SUBXrs:
2592 return AArch64::SUBSXrs;
2593 case AArch64::SUBXrx:
2594 return AArch64::SUBSXrx;
2596 case AArch64::AND_PPzPP:
2597 return AArch64::ANDS_PPzPP;
2598 case AArch64::BIC_PPzPP:
2599 return AArch64::BICS_PPzPP;
2600 case AArch64::EOR_PPzPP:
2601 return AArch64::EORS_PPzPP;
2602 case AArch64::NAND_PPzPP:
2603 return AArch64::NANDS_PPzPP;
2604 case AArch64::NOR_PPzPP:
2605 return AArch64::NORS_PPzPP;
2606 case AArch64::ORN_PPzPP:
2607 return AArch64::ORNS_PPzPP;
2608 case AArch64::ORR_PPzPP:
2609 return AArch64::ORRS_PPzPP;
2610 case AArch64::BRKA_PPzP:
2611 return AArch64::BRKAS_PPzP;
2612 case AArch64::BRKPA_PPzPP:
2613 return AArch64::BRKPAS_PPzPP;
2614 case AArch64::BRKB_PPzP:
2615 return AArch64::BRKBS_PPzP;
2616 case AArch64::BRKPB_PPzPP:
2617 return AArch64::BRKPBS_PPzPP;
2618 case AArch64::BRKN_PPzP:
2619 return AArch64::BRKNS_PPzP;
2620 case AArch64::RDFFR_PPz:
2621 return AArch64::RDFFRS_PPz;
2622 case AArch64::PTRUE_B:
2623 return AArch64::PTRUES_B;
2634 if (
MI.hasOrderedMemoryRef())
2639 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2640 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2641 "Expected a reg or frame index operand.");
2645 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2647 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2660 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2661 Register BaseReg =
MI.getOperand(1).getReg();
2663 if (
MI.modifiesRegister(BaseReg,
TRI))
2676 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2678 MI.getMF()->getFunction().needsUnwindTableEntry();
2684 if (Subtarget.isPaired128Slow()) {
2685 switch (
MI.getOpcode()) {
2688 case AArch64::LDURQi:
2689 case AArch64::STURQi:
2690 case AArch64::LDRQui:
2691 case AArch64::STRQui:
2718std::optional<ExtAddrMode>
2723 bool OffsetIsScalable;
2724 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2725 return std::nullopt;
2728 return std::nullopt;
2743 int64_t OffsetScale = 1;
2748 case AArch64::LDURQi:
2749 case AArch64::STURQi:
2753 case AArch64::LDURDi:
2754 case AArch64::STURDi:
2755 case AArch64::LDURXi:
2756 case AArch64::STURXi:
2760 case AArch64::LDURWi:
2761 case AArch64::LDURSWi:
2762 case AArch64::STURWi:
2766 case AArch64::LDURHi:
2767 case AArch64::STURHi:
2768 case AArch64::LDURHHi:
2769 case AArch64::STURHHi:
2770 case AArch64::LDURSHXi:
2771 case AArch64::LDURSHWi:
2775 case AArch64::LDRBroX:
2776 case AArch64::LDRBBroX:
2777 case AArch64::LDRSBXroX:
2778 case AArch64::LDRSBWroX:
2779 case AArch64::STRBroX:
2780 case AArch64::STRBBroX:
2781 case AArch64::LDURBi:
2782 case AArch64::LDURBBi:
2783 case AArch64::LDURSBXi:
2784 case AArch64::LDURSBWi:
2785 case AArch64::STURBi:
2786 case AArch64::STURBBi:
2787 case AArch64::LDRBui:
2788 case AArch64::LDRBBui:
2789 case AArch64::LDRSBXui:
2790 case AArch64::LDRSBWui:
2791 case AArch64::STRBui:
2792 case AArch64::STRBBui:
2796 case AArch64::LDRQroX:
2797 case AArch64::STRQroX:
2798 case AArch64::LDRQui:
2799 case AArch64::STRQui:
2804 case AArch64::LDRDroX:
2805 case AArch64::STRDroX:
2806 case AArch64::LDRXroX:
2807 case AArch64::STRXroX:
2808 case AArch64::LDRDui:
2809 case AArch64::STRDui:
2810 case AArch64::LDRXui:
2811 case AArch64::STRXui:
2816 case AArch64::LDRWroX:
2817 case AArch64::LDRSWroX:
2818 case AArch64::STRWroX:
2819 case AArch64::LDRWui:
2820 case AArch64::LDRSWui:
2821 case AArch64::STRWui:
2826 case AArch64::LDRHroX:
2827 case AArch64::STRHroX:
2828 case AArch64::LDRHHroX:
2829 case AArch64::STRHHroX:
2830 case AArch64::LDRSHXroX:
2831 case AArch64::LDRSHWroX:
2832 case AArch64::LDRHui:
2833 case AArch64::STRHui:
2834 case AArch64::LDRHHui:
2835 case AArch64::STRHHui:
2836 case AArch64::LDRSHXui:
2837 case AArch64::LDRSHWui:
2845 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
2869 case AArch64::SBFMXri:
2882 AM.
Scale = OffsetScale;
2887 case TargetOpcode::SUBREG_TO_REG: {
2900 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
2904 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
2906 DefMI.getOperand(3).getImm() != 0)
2913 AM.
Scale = OffsetScale;
2924 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
2925 int64_t NewOffset) ->
bool {
2926 int64_t MinOffset, MaxOffset;
2943 return OldOffset < MinOffset || OldOffset > MaxOffset ||
2944 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
2946 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
2948 int64_t NewOffset = OldOffset + Disp;
2953 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
2963 auto canFoldAddRegIntoAddrMode =
2980 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
2981 Subtarget.isSTRQroSlow();
2990 case AArch64::ADDXri:
2996 return canFoldAddSubImmIntoAddrMode(Disp);
2998 case AArch64::SUBXri:
3004 return canFoldAddSubImmIntoAddrMode(-Disp);
3006 case AArch64::ADDXrs: {
3019 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3021 if (avoidSlowSTRQ(MemI))
3024 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3027 case AArch64::ADDXrr:
3035 if (!OptSize && avoidSlowSTRQ(MemI))
3037 return canFoldAddRegIntoAddrMode(1);
3039 case AArch64::ADDXrx:
3047 if (!OptSize && avoidSlowSTRQ(MemI))
3056 return canFoldAddRegIntoAddrMode(
3071 case AArch64::LDURQi:
3072 case AArch64::LDRQui:
3073 return AArch64::LDRQroX;
3074 case AArch64::STURQi:
3075 case AArch64::STRQui:
3076 return AArch64::STRQroX;
3077 case AArch64::LDURDi:
3078 case AArch64::LDRDui:
3079 return AArch64::LDRDroX;
3080 case AArch64::STURDi:
3081 case AArch64::STRDui:
3082 return AArch64::STRDroX;
3083 case AArch64::LDURXi:
3084 case AArch64::LDRXui:
3085 return AArch64::LDRXroX;
3086 case AArch64::STURXi:
3087 case AArch64::STRXui:
3088 return AArch64::STRXroX;
3089 case AArch64::LDURWi:
3090 case AArch64::LDRWui:
3091 return AArch64::LDRWroX;
3092 case AArch64::LDURSWi:
3093 case AArch64::LDRSWui:
3094 return AArch64::LDRSWroX;
3095 case AArch64::STURWi:
3096 case AArch64::STRWui:
3097 return AArch64::STRWroX;
3098 case AArch64::LDURHi:
3099 case AArch64::LDRHui:
3100 return AArch64::LDRHroX;
3101 case AArch64::STURHi:
3102 case AArch64::STRHui:
3103 return AArch64::STRHroX;
3104 case AArch64::LDURHHi:
3105 case AArch64::LDRHHui:
3106 return AArch64::LDRHHroX;
3107 case AArch64::STURHHi:
3108 case AArch64::STRHHui:
3109 return AArch64::STRHHroX;
3110 case AArch64::LDURSHXi:
3111 case AArch64::LDRSHXui:
3112 return AArch64::LDRSHXroX;
3113 case AArch64::LDURSHWi:
3114 case AArch64::LDRSHWui:
3115 return AArch64::LDRSHWroX;
3116 case AArch64::LDURBi:
3117 case AArch64::LDRBui:
3118 return AArch64::LDRBroX;
3119 case AArch64::LDURBBi:
3120 case AArch64::LDRBBui:
3121 return AArch64::LDRBBroX;
3122 case AArch64::LDURSBXi:
3123 case AArch64::LDRSBXui:
3124 return AArch64::LDRSBXroX;
3125 case AArch64::LDURSBWi:
3126 case AArch64::LDRSBWui:
3127 return AArch64::LDRSBWroX;
3128 case AArch64::STURBi:
3129 case AArch64::STRBui:
3130 return AArch64::STRBroX;
3131 case AArch64::STURBBi:
3132 case AArch64::STRBBui:
3133 return AArch64::STRBBroX;
3145 case AArch64::LDURQi:
3147 return AArch64::LDRQui;
3148 case AArch64::STURQi:
3150 return AArch64::STRQui;
3151 case AArch64::LDURDi:
3153 return AArch64::LDRDui;
3154 case AArch64::STURDi:
3156 return AArch64::STRDui;
3157 case AArch64::LDURXi:
3159 return AArch64::LDRXui;
3160 case AArch64::STURXi:
3162 return AArch64::STRXui;
3163 case AArch64::LDURWi:
3165 return AArch64::LDRWui;
3166 case AArch64::LDURSWi:
3168 return AArch64::LDRSWui;
3169 case AArch64::STURWi:
3171 return AArch64::STRWui;
3172 case AArch64::LDURHi:
3174 return AArch64::LDRHui;
3175 case AArch64::STURHi:
3177 return AArch64::STRHui;
3178 case AArch64::LDURHHi:
3180 return AArch64::LDRHHui;
3181 case AArch64::STURHHi:
3183 return AArch64::STRHHui;
3184 case AArch64::LDURSHXi:
3186 return AArch64::LDRSHXui;
3187 case AArch64::LDURSHWi:
3189 return AArch64::LDRSHWui;
3190 case AArch64::LDURBi:
3192 return AArch64::LDRBui;
3193 case AArch64::LDURBBi:
3195 return AArch64::LDRBBui;
3196 case AArch64::LDURSBXi:
3198 return AArch64::LDRSBXui;
3199 case AArch64::LDURSBWi:
3201 return AArch64::LDRSBWui;
3202 case AArch64::STURBi:
3204 return AArch64::STRBui;
3205 case AArch64::STURBBi:
3207 return AArch64::STRBBui;
3208 case AArch64::LDRQui:
3209 case AArch64::STRQui:
3212 case AArch64::LDRDui:
3213 case AArch64::STRDui:
3214 case AArch64::LDRXui:
3215 case AArch64::STRXui:
3218 case AArch64::LDRWui:
3219 case AArch64::LDRSWui:
3220 case AArch64::STRWui:
3223 case AArch64::LDRHui:
3224 case AArch64::STRHui:
3225 case AArch64::LDRHHui:
3226 case AArch64::STRHHui:
3227 case AArch64::LDRSHXui:
3228 case AArch64::LDRSHWui:
3231 case AArch64::LDRBui:
3232 case AArch64::LDRBBui:
3233 case AArch64::LDRSBXui:
3234 case AArch64::LDRSBWui:
3235 case AArch64::STRBui:
3236 case AArch64::STRBBui:
3250 case AArch64::LDURQi:
3251 case AArch64::STURQi:
3252 case AArch64::LDURDi:
3253 case AArch64::STURDi:
3254 case AArch64::LDURXi:
3255 case AArch64::STURXi:
3256 case AArch64::LDURWi:
3257 case AArch64::LDURSWi:
3258 case AArch64::STURWi:
3259 case AArch64::LDURHi:
3260 case AArch64::STURHi:
3261 case AArch64::LDURHHi:
3262 case AArch64::STURHHi:
3263 case AArch64::LDURSHXi:
3264 case AArch64::LDURSHWi:
3265 case AArch64::LDURBi:
3266 case AArch64::STURBi:
3267 case AArch64::LDURBBi:
3268 case AArch64::STURBBi:
3269 case AArch64::LDURSBWi:
3270 case AArch64::LDURSBXi:
3272 case AArch64::LDRQui:
3273 return AArch64::LDURQi;
3274 case AArch64::STRQui:
3275 return AArch64::STURQi;
3276 case AArch64::LDRDui:
3277 return AArch64::LDURDi;
3278 case AArch64::STRDui:
3279 return AArch64::STURDi;
3280 case AArch64::LDRXui:
3281 return AArch64::LDURXi;
3282 case AArch64::STRXui:
3283 return AArch64::STURXi;
3284 case AArch64::LDRWui:
3285 return AArch64::LDURWi;
3286 case AArch64::LDRSWui:
3287 return AArch64::LDURSWi;
3288 case AArch64::STRWui:
3289 return AArch64::STURWi;
3290 case AArch64::LDRHui:
3291 return AArch64::LDURHi;
3292 case AArch64::STRHui:
3293 return AArch64::STURHi;
3294 case AArch64::LDRHHui:
3295 return AArch64::LDURHHi;
3296 case AArch64::STRHHui:
3297 return AArch64::STURHHi;
3298 case AArch64::LDRSHXui:
3299 return AArch64::LDURSHXi;
3300 case AArch64::LDRSHWui:
3301 return AArch64::LDURSHWi;
3302 case AArch64::LDRBBui:
3303 return AArch64::LDURBBi;
3304 case AArch64::LDRBui:
3305 return AArch64::LDURBi;
3306 case AArch64::STRBBui:
3307 return AArch64::STURBBi;
3308 case AArch64::STRBui:
3309 return AArch64::STURBi;
3310 case AArch64::LDRSBWui:
3311 return AArch64::LDURSBWi;
3312 case AArch64::LDRSBXui:
3313 return AArch64::LDURSBXi;
3326 case AArch64::LDRQroX:
3327 case AArch64::LDURQi:
3328 case AArch64::LDRQui:
3329 return AArch64::LDRQroW;
3330 case AArch64::STRQroX:
3331 case AArch64::STURQi:
3332 case AArch64::STRQui:
3333 return AArch64::STRQroW;
3334 case AArch64::LDRDroX:
3335 case AArch64::LDURDi:
3336 case AArch64::LDRDui:
3337 return AArch64::LDRDroW;
3338 case AArch64::STRDroX:
3339 case AArch64::STURDi:
3340 case AArch64::STRDui:
3341 return AArch64::STRDroW;
3342 case AArch64::LDRXroX:
3343 case AArch64::LDURXi:
3344 case AArch64::LDRXui:
3345 return AArch64::LDRXroW;
3346 case AArch64::STRXroX:
3347 case AArch64::STURXi:
3348 case AArch64::STRXui:
3349 return AArch64::STRXroW;
3350 case AArch64::LDRWroX:
3351 case AArch64::LDURWi:
3352 case AArch64::LDRWui:
3353 return AArch64::LDRWroW;
3354 case AArch64::LDRSWroX:
3355 case AArch64::LDURSWi:
3356 case AArch64::LDRSWui:
3357 return AArch64::LDRSWroW;
3358 case AArch64::STRWroX:
3359 case AArch64::STURWi:
3360 case AArch64::STRWui:
3361 return AArch64::STRWroW;
3362 case AArch64::LDRHroX:
3363 case AArch64::LDURHi:
3364 case AArch64::LDRHui:
3365 return AArch64::LDRHroW;
3366 case AArch64::STRHroX:
3367 case AArch64::STURHi:
3368 case AArch64::STRHui:
3369 return AArch64::STRHroW;
3370 case AArch64::LDRHHroX:
3371 case AArch64::LDURHHi:
3372 case AArch64::LDRHHui:
3373 return AArch64::LDRHHroW;
3374 case AArch64::STRHHroX:
3375 case AArch64::STURHHi:
3376 case AArch64::STRHHui:
3377 return AArch64::STRHHroW;
3378 case AArch64::LDRSHXroX:
3379 case AArch64::LDURSHXi:
3380 case AArch64::LDRSHXui:
3381 return AArch64::LDRSHXroW;
3382 case AArch64::LDRSHWroX:
3383 case AArch64::LDURSHWi:
3384 case AArch64::LDRSHWui:
3385 return AArch64::LDRSHWroW;
3386 case AArch64::LDRBroX:
3387 case AArch64::LDURBi:
3388 case AArch64::LDRBui:
3389 return AArch64::LDRBroW;
3390 case AArch64::LDRBBroX:
3391 case AArch64::LDURBBi:
3392 case AArch64::LDRBBui:
3393 return AArch64::LDRBBroW;
3394 case AArch64::LDRSBXroX:
3395 case AArch64::LDURSBXi:
3396 case AArch64::LDRSBXui:
3397 return AArch64::LDRSBXroW;
3398 case AArch64::LDRSBWroX:
3399 case AArch64::LDURSBWi:
3400 case AArch64::LDRSBWui:
3401 return AArch64::LDRSBWroW;
3402 case AArch64::STRBroX:
3403 case AArch64::STURBi:
3404 case AArch64::STRBui:
3405 return AArch64::STRBroW;
3406 case AArch64::STRBBroX:
3407 case AArch64::STURBBi:
3408 case AArch64::STRBBui:
3409 return AArch64::STRBBroW;
3424 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3434 return B.getInstr();
3438 "Addressing mode not supported for folding");
3455 return B.getInstr();
3462 "Address offset can be a register or an immediate, but not both");
3464 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3469 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3483 return B.getInstr();
3487 "Function must not be called with an addressing mode it can't handle");
3492 bool &OffsetIsScalable,
TypeSize &Width,
3513 int64_t Dummy1, Dummy2;
3532 if (!BaseOp->
isReg() && !BaseOp->
isFI())
3542 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3547 TypeSize &Width, int64_t &MinOffset,
3548 int64_t &MaxOffset) {
3554 MinOffset = MaxOffset = 0;
3557 case AArch64::LDRQui:
3558 case AArch64::STRQui:
3564 case AArch64::LDRXui:
3565 case AArch64::LDRDui:
3566 case AArch64::STRXui:
3567 case AArch64::STRDui:
3568 case AArch64::PRFMui:
3574 case AArch64::LDRWui:
3575 case AArch64::LDRSui:
3576 case AArch64::LDRSWui:
3577 case AArch64::STRWui:
3578 case AArch64::STRSui:
3584 case AArch64::LDRHui:
3585 case AArch64::LDRHHui:
3586 case AArch64::LDRSHWui:
3587 case AArch64::LDRSHXui:
3588 case AArch64::STRHui:
3589 case AArch64::STRHHui:
3595 case AArch64::LDRBui:
3596 case AArch64::LDRBBui:
3597 case AArch64::LDRSBWui:
3598 case AArch64::LDRSBXui:
3599 case AArch64::STRBui:
3600 case AArch64::STRBBui:
3607 case AArch64::STRQpre:
3608 case AArch64::LDRQpost:
3614 case AArch64::STRXpre:
3615 case AArch64::STRDpre:
3616 case AArch64::LDRXpost:
3617 case AArch64::LDRDpost:
3623 case AArch64::STRWpost:
3624 case AArch64::LDRWpost:
3631 case AArch64::LDURQi:
3632 case AArch64::STURQi:
3638 case AArch64::LDURXi:
3639 case AArch64::LDURDi:
3640 case AArch64::LDAPURXi:
3641 case AArch64::STURXi:
3642 case AArch64::STURDi:
3643 case AArch64::STLURXi:
3644 case AArch64::PRFUMi:
3650 case AArch64::LDURWi:
3651 case AArch64::LDURSi:
3652 case AArch64::LDURSWi:
3653 case AArch64::LDAPURi:
3654 case AArch64::LDAPURSWi:
3655 case AArch64::STURWi:
3656 case AArch64::STURSi:
3657 case AArch64::STLURWi:
3663 case AArch64::LDURHi:
3664 case AArch64::LDURHHi:
3665 case AArch64::LDURSHXi:
3666 case AArch64::LDURSHWi:
3667 case AArch64::LDAPURHi:
3668 case AArch64::LDAPURSHWi:
3669 case AArch64::LDAPURSHXi:
3670 case AArch64::STURHi:
3671 case AArch64::STURHHi:
3672 case AArch64::STLURHi:
3678 case AArch64::LDURBi:
3679 case AArch64::LDURBBi:
3680 case AArch64::LDURSBXi:
3681 case AArch64::LDURSBWi:
3682 case AArch64::LDAPURBi:
3683 case AArch64::LDAPURSBWi:
3684 case AArch64::LDAPURSBXi:
3685 case AArch64::STURBi:
3686 case AArch64::STURBBi:
3687 case AArch64::STLURBi:
3694 case AArch64::LDPQi:
3695 case AArch64::LDNPQi:
3696 case AArch64::STPQi:
3697 case AArch64::STNPQi:
3703 case AArch64::LDPXi:
3704 case AArch64::LDPDi:
3705 case AArch64::LDNPXi:
3706 case AArch64::LDNPDi:
3707 case AArch64::STPXi:
3708 case AArch64::STPDi:
3709 case AArch64::STNPXi:
3710 case AArch64::STNPDi:
3716 case AArch64::LDPWi:
3717 case AArch64::LDPSi:
3718 case AArch64::LDNPWi:
3719 case AArch64::LDNPSi:
3720 case AArch64::STPWi:
3721 case AArch64::STPSi:
3722 case AArch64::STNPWi:
3723 case AArch64::STNPSi:
3730 case AArch64::STPQpre:
3731 case AArch64::LDPQpost:
3737 case AArch64::STPXpre:
3738 case AArch64::LDPXpost:
3739 case AArch64::STPDpre:
3740 case AArch64::LDPDpost:
3746 case AArch64::StoreSwiftAsyncContext:
3759 case AArch64::TAGPstack:
3769 case AArch64::STZGi:
3776 case AArch64::STR_ZZZZXI:
3777 case AArch64::LDR_ZZZZXI:
3783 case AArch64::STR_ZZZXI:
3784 case AArch64::LDR_ZZZXI:
3790 case AArch64::STR_ZZXI:
3791 case AArch64::LDR_ZZXI:
3797 case AArch64::LDR_PXI:
3798 case AArch64::STR_PXI:
3804 case AArch64::LDR_PPXI:
3805 case AArch64::STR_PPXI:
3811 case AArch64::LDR_ZXI:
3812 case AArch64::STR_ZXI:
3818 case AArch64::LD1B_IMM:
3819 case AArch64::LD1H_IMM:
3820 case AArch64::LD1W_IMM:
3821 case AArch64::LD1D_IMM:
3822 case AArch64::LDNT1B_ZRI:
3823 case AArch64::LDNT1H_ZRI:
3824 case AArch64::LDNT1W_ZRI:
3825 case AArch64::LDNT1D_ZRI:
3826 case AArch64::ST1B_IMM:
3827 case AArch64::ST1H_IMM:
3828 case AArch64::ST1W_IMM:
3829 case AArch64::ST1D_IMM:
3830 case AArch64::STNT1B_ZRI:
3831 case AArch64::STNT1H_ZRI:
3832 case AArch64::STNT1W_ZRI:
3833 case AArch64::STNT1D_ZRI:
3834 case AArch64::LDNF1B_IMM:
3835 case AArch64::LDNF1H_IMM:
3836 case AArch64::LDNF1W_IMM:
3837 case AArch64::LDNF1D_IMM:
3845 case AArch64::LD2B_IMM:
3846 case AArch64::LD2H_IMM:
3847 case AArch64::LD2W_IMM:
3848 case AArch64::LD2D_IMM:
3849 case AArch64::ST2B_IMM:
3850 case AArch64::ST2H_IMM:
3851 case AArch64::ST2W_IMM:
3852 case AArch64::ST2D_IMM:
3858 case AArch64::LD3B_IMM:
3859 case AArch64::LD3H_IMM:
3860 case AArch64::LD3W_IMM:
3861 case AArch64::LD3D_IMM:
3862 case AArch64::ST3B_IMM:
3863 case AArch64::ST3H_IMM:
3864 case AArch64::ST3W_IMM:
3865 case AArch64::ST3D_IMM:
3871 case AArch64::LD4B_IMM:
3872 case AArch64::LD4H_IMM:
3873 case AArch64::LD4W_IMM:
3874 case AArch64::LD4D_IMM:
3875 case AArch64::ST4B_IMM:
3876 case AArch64::ST4H_IMM:
3877 case AArch64::ST4W_IMM:
3878 case AArch64::ST4D_IMM:
3884 case AArch64::LD1B_H_IMM:
3885 case AArch64::LD1SB_H_IMM:
3886 case AArch64::LD1H_S_IMM:
3887 case AArch64::LD1SH_S_IMM:
3888 case AArch64::LD1W_D_IMM:
3889 case AArch64::LD1SW_D_IMM:
3890 case AArch64::ST1B_H_IMM:
3891 case AArch64::ST1H_S_IMM:
3892 case AArch64::ST1W_D_IMM:
3893 case AArch64::LDNF1B_H_IMM:
3894 case AArch64::LDNF1SB_H_IMM:
3895 case AArch64::LDNF1H_S_IMM:
3896 case AArch64::LDNF1SH_S_IMM:
3897 case AArch64::LDNF1W_D_IMM:
3898 case AArch64::LDNF1SW_D_IMM:
3906 case AArch64::LD1B_S_IMM:
3907 case AArch64::LD1SB_S_IMM:
3908 case AArch64::LD1H_D_IMM:
3909 case AArch64::LD1SH_D_IMM:
3910 case AArch64::ST1B_S_IMM:
3911 case AArch64::ST1H_D_IMM:
3912 case AArch64::LDNF1B_S_IMM:
3913 case AArch64::LDNF1SB_S_IMM:
3914 case AArch64::LDNF1H_D_IMM:
3915 case AArch64::LDNF1SH_D_IMM:
3923 case AArch64::LD1B_D_IMM:
3924 case AArch64::LD1SB_D_IMM:
3925 case AArch64::ST1B_D_IMM:
3926 case AArch64::LDNF1B_D_IMM:
3927 case AArch64::LDNF1SB_D_IMM:
3935 case AArch64::ST2Gi:
3936 case AArch64::STZ2Gi:
3942 case AArch64::STGPi:
3948 case AArch64::LD1RB_IMM:
3949 case AArch64::LD1RB_H_IMM:
3950 case AArch64::LD1RB_S_IMM:
3951 case AArch64::LD1RB_D_IMM:
3952 case AArch64::LD1RSB_H_IMM:
3953 case AArch64::LD1RSB_S_IMM:
3954 case AArch64::LD1RSB_D_IMM:
3960 case AArch64::LD1RH_IMM:
3961 case AArch64::LD1RH_S_IMM:
3962 case AArch64::LD1RH_D_IMM:
3963 case AArch64::LD1RSH_S_IMM:
3964 case AArch64::LD1RSH_D_IMM:
3970 case AArch64::LD1RW_IMM:
3971 case AArch64::LD1RW_D_IMM:
3972 case AArch64::LD1RSW_IMM:
3978 case AArch64::LD1RD_IMM:
3994 case AArch64::LDRBBui:
3995 case AArch64::LDURBBi:
3996 case AArch64::LDRSBWui:
3997 case AArch64::LDURSBWi:
3998 case AArch64::STRBBui:
3999 case AArch64::STURBBi:
4001 case AArch64::LDRHHui:
4002 case AArch64::LDURHHi:
4003 case AArch64::LDRSHWui:
4004 case AArch64::LDURSHWi:
4005 case AArch64::STRHHui:
4006 case AArch64::STURHHi:
4008 case AArch64::LDRSui:
4009 case AArch64::LDURSi:
4010 case AArch64::LDRSpre:
4011 case AArch64::LDRSWui:
4012 case AArch64::LDURSWi:
4013 case AArch64::LDRSWpre:
4014 case AArch64::LDRWpre:
4015 case AArch64::LDRWui:
4016 case AArch64::LDURWi:
4017 case AArch64::STRSui:
4018 case AArch64::STURSi:
4019 case AArch64::STRSpre:
4020 case AArch64::STRWui:
4021 case AArch64::STURWi:
4022 case AArch64::STRWpre:
4023 case AArch64::LDPSi:
4024 case AArch64::LDPSWi:
4025 case AArch64::LDPWi:
4026 case AArch64::STPSi:
4027 case AArch64::STPWi:
4029 case AArch64::LDRDui:
4030 case AArch64::LDURDi:
4031 case AArch64::LDRDpre:
4032 case AArch64::LDRXui:
4033 case AArch64::LDURXi:
4034 case AArch64::LDRXpre:
4035 case AArch64::STRDui:
4036 case AArch64::STURDi:
4037 case AArch64::STRDpre:
4038 case AArch64::STRXui:
4039 case AArch64::STURXi:
4040 case AArch64::STRXpre:
4041 case AArch64::LDPDi:
4042 case AArch64::LDPXi:
4043 case AArch64::STPDi:
4044 case AArch64::STPXi:
4046 case AArch64::LDRQui:
4047 case AArch64::LDURQi:
4048 case AArch64::STRQui:
4049 case AArch64::STURQi:
4050 case AArch64::STRQpre:
4051 case AArch64::LDPQi:
4052 case AArch64::LDRQpre:
4053 case AArch64::STPQi:
4055 case AArch64::STZGi:
4056 case AArch64::ST2Gi:
4057 case AArch64::STZ2Gi:
4058 case AArch64::STGPi:
4064 switch (
MI.getOpcode()) {
4067 case AArch64::LDRWpre:
4068 case AArch64::LDRXpre:
4069 case AArch64::LDRSWpre:
4070 case AArch64::LDRSpre:
4071 case AArch64::LDRDpre:
4072 case AArch64::LDRQpre:
4078 switch (
MI.getOpcode()) {
4081 case AArch64::STRWpre:
4082 case AArch64::STRXpre:
4083 case AArch64::STRSpre:
4084 case AArch64::STRDpre:
4085 case AArch64::STRQpre:
4095 switch (
MI.getOpcode()) {
4098 case AArch64::LDPSi:
4099 case AArch64::LDPSWi:
4100 case AArch64::LDPDi:
4101 case AArch64::LDPQi:
4102 case AArch64::LDPWi:
4103 case AArch64::LDPXi:
4104 case AArch64::STPSi:
4105 case AArch64::STPDi:
4106 case AArch64::STPQi:
4107 case AArch64::STPWi:
4108 case AArch64::STPXi:
4109 case AArch64::STGPi:
4118 return MI.getOperand(
Idx);
4126 return MI.getOperand(
Idx);
4131 if (
MI.getParent() ==
nullptr)
4141 auto Reg =
Op.getReg();
4142 if (Reg.isPhysical())
4143 return AArch64::FPR16RegClass.
contains(Reg);
4145 return TRC == &AArch64::FPR16RegClass ||
4146 TRC == &AArch64::FPR16_loRegClass;
4155 auto Reg =
Op.getReg();
4156 if (Reg.isPhysical())
4157 return AArch64::FPR128RegClass.
contains(Reg);
4159 return TRC == &AArch64::FPR128RegClass ||
4160 TRC == &AArch64::FPR128_loRegClass;
4166 switch (
MI.getOpcode()) {
4169 case AArch64::PACIASP:
4170 case AArch64::PACIBSP:
4173 case AArch64::PAUTH_PROLOGUE:
4176 case AArch64::HINT: {
4177 unsigned Imm =
MI.getOperand(0).getImm();
4179 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4182 if (Imm == 25 || Imm == 27)
4194 assert(Reg.isPhysical() &&
"Expected physical register in isFpOrNEON");
4195 return AArch64::FPR128RegClass.contains(Reg) ||
4196 AArch64::FPR64RegClass.contains(Reg) ||
4197 AArch64::FPR32RegClass.contains(Reg) ||
4198 AArch64::FPR16RegClass.contains(Reg) ||
4199 AArch64::FPR8RegClass.contains(Reg);
4206 auto Reg =
Op.getReg();
4207 if (Reg.isPhysical())
4211 return TRC == &AArch64::FPR128RegClass ||
4212 TRC == &AArch64::FPR128_loRegClass ||
4213 TRC == &AArch64::FPR64RegClass ||
4214 TRC == &AArch64::FPR64_loRegClass ||
4215 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4216 TRC == &AArch64::FPR8RegClass;
4238 if (FirstOpc == SecondOpc)
4244 case AArch64::STRSui:
4245 case AArch64::STURSi:
4246 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4247 case AArch64::STRDui:
4248 case AArch64::STURDi:
4249 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4250 case AArch64::STRQui:
4251 case AArch64::STURQi:
4252 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4253 case AArch64::STRWui:
4254 case AArch64::STURWi:
4255 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4256 case AArch64::STRXui:
4257 case AArch64::STURXi:
4258 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4259 case AArch64::LDRSui:
4260 case AArch64::LDURSi:
4261 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4262 case AArch64::LDRDui:
4263 case AArch64::LDURDi:
4264 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4265 case AArch64::LDRQui:
4266 case AArch64::LDURQi:
4267 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4268 case AArch64::LDRWui:
4269 case AArch64::LDURWi:
4270 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4271 case AArch64::LDRSWui:
4272 case AArch64::LDURSWi:
4273 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4274 case AArch64::LDRXui:
4275 case AArch64::LDURXi:
4276 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4283 int64_t Offset1,
unsigned Opcode1,
int FI2,
4284 int64_t Offset2,
unsigned Opcode2) {
4290 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4293 if (ObjectOffset1 % Scale1 != 0)
4295 ObjectOffset1 /= Scale1;
4297 if (ObjectOffset2 % Scale2 != 0)
4299 ObjectOffset2 /= Scale2;
4300 ObjectOffset1 += Offset1;
4301 ObjectOffset2 += Offset2;
4302 return ObjectOffset1 + 1 == ObjectOffset2;
4314 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4315 unsigned NumBytes)
const {
4325 "Only base registers and frame indices are supported.");
4332 if (ClusterSize > 2)
4339 unsigned FirstOpc = FirstLdSt.
getOpcode();
4340 unsigned SecondOpc = SecondLdSt.
getOpcode();
4360 if (Offset1 > 63 || Offset1 < -64)
4365 if (BaseOp1.
isFI()) {
4367 "Caller should have ordered offsets.");
4372 BaseOp2.
getIndex(), Offset2, SecondOpc);
4375 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4377 return Offset1 + 1 == Offset2;
4381 unsigned Reg,
unsigned SubIdx,
4385 return MIB.
addReg(Reg, State);
4388 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4389 return MIB.
addReg(Reg, State, SubIdx);
4396 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4405 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4407 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4408 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4409 unsigned NumRegs = Indices.
size();
4411 int SubReg = 0,
End = NumRegs, Incr = 1;
4429 unsigned SrcReg,
bool KillSrc,
4430 unsigned Opcode,
unsigned ZeroReg,
4433 unsigned NumRegs = Indices.
size();
4436 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4437 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4438 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4439 "GPR reg sequences should not be able to overlap");
4455 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4456 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4459 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4461 if (Subtarget.hasZeroCycleRegMove()) {
4464 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4466 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4482 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4487 if (Subtarget.hasZeroCycleRegMove()) {
4490 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4492 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4512 if (AArch64::PPRRegClass.
contains(DestReg) &&
4513 AArch64::PPRRegClass.
contains(SrcReg)) {
4515 "Unexpected SVE register.");
4525 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4526 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4527 if (DestIsPNR || SrcIsPNR) {
4529 return (R - AArch64::PN0) + AArch64::P0;
4531 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4532 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4534 if (PPRSrcReg != PPRDestReg) {
4546 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4547 AArch64::ZPRRegClass.
contains(SrcReg)) {
4549 "Unexpected SVE register.");
4557 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
4558 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
4559 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
4560 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
4562 "Unexpected SVE register.");
4563 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
4570 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
4571 AArch64::ZPR3RegClass.
contains(SrcReg)) {
4573 "Unexpected SVE register.");
4574 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4582 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
4583 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
4584 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
4585 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
4587 "Unexpected SVE register.");
4588 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4589 AArch64::zsub2, AArch64::zsub3};
4595 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
4596 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
4597 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
4603 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
4617 if (AArch64::DDDDRegClass.
contains(DestReg) &&
4618 AArch64::DDDDRegClass.
contains(SrcReg)) {
4619 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4620 AArch64::dsub2, AArch64::dsub3};
4627 if (AArch64::DDDRegClass.
contains(DestReg) &&
4628 AArch64::DDDRegClass.
contains(SrcReg)) {
4629 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4637 if (AArch64::DDRegClass.
contains(DestReg) &&
4638 AArch64::DDRegClass.
contains(SrcReg)) {
4639 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
4646 if (AArch64::QQQQRegClass.
contains(DestReg) &&
4647 AArch64::QQQQRegClass.
contains(SrcReg)) {
4648 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4649 AArch64::qsub2, AArch64::qsub3};
4656 if (AArch64::QQQRegClass.
contains(DestReg) &&
4657 AArch64::QQQRegClass.
contains(SrcReg)) {
4658 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4666 if (AArch64::QQRegClass.
contains(DestReg) &&
4667 AArch64::QQRegClass.
contains(SrcReg)) {
4668 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
4674 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
4675 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
4676 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
4678 AArch64::XZR, Indices);
4682 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
4683 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
4684 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
4686 AArch64::WZR, Indices);
4690 if (AArch64::FPR128RegClass.
contains(DestReg) &&
4691 AArch64::FPR128RegClass.
contains(SrcReg)) {
4696 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
4697 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
4717 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4718 AArch64::FPR64RegClass.
contains(SrcReg)) {
4724 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4725 AArch64::FPR32RegClass.
contains(SrcReg)) {
4731 if (AArch64::FPR16RegClass.
contains(DestReg) &&
4732 AArch64::FPR16RegClass.
contains(SrcReg)) {
4734 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
4736 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
4742 if (AArch64::FPR8RegClass.
contains(DestReg) &&
4743 AArch64::FPR8RegClass.
contains(SrcReg)) {
4745 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
4747 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
4754 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4755 AArch64::GPR64RegClass.
contains(SrcReg)) {
4760 if (AArch64::GPR64RegClass.
contains(DestReg) &&
4761 AArch64::FPR64RegClass.
contains(SrcReg)) {
4767 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4768 AArch64::GPR32RegClass.
contains(SrcReg)) {
4773 if (AArch64::GPR32RegClass.
contains(DestReg) &&
4774 AArch64::FPR32RegClass.
contains(SrcReg)) {
4780 if (DestReg == AArch64::NZCV) {
4781 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
4783 .
addImm(AArch64SysReg::NZCV)
4789 if (SrcReg == AArch64::NZCV) {
4790 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
4792 .
addImm(AArch64SysReg::NZCV)
4799 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
4800 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
4810 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
4815 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
4817 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
4830 Register SrcReg,
bool isKill,
int FI,
4845 switch (
TRI->getSpillSize(*RC)) {
4847 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4848 Opc = AArch64::STRBui;
4851 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4852 Opc = AArch64::STRHui;
4853 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
4854 AArch64::PPRRegClass.hasSubClassEq(RC)) {
4856 "Unexpected register store without SVE store instructions");
4857 Opc = AArch64::STR_PXI;
4863 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4864 Opc = AArch64::STRWui;
4868 assert(SrcReg != AArch64::WSP);
4869 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4870 Opc = AArch64::STRSui;
4871 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
4872 Opc = AArch64::STR_PPXI;
4877 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4878 Opc = AArch64::STRXui;
4882 assert(SrcReg != AArch64::SP);
4883 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4884 Opc = AArch64::STRDui;
4885 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4887 get(AArch64::STPWi), SrcReg, isKill,
4888 AArch64::sube32, AArch64::subo32, FI, MMO);
4893 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4894 Opc = AArch64::STRQui;
4895 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4896 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4897 Opc = AArch64::ST1Twov1d;
4899 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4901 get(AArch64::STPXi), SrcReg, isKill,
4902 AArch64::sube64, AArch64::subo64, FI, MMO);
4904 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4906 "Unexpected register store without SVE store instructions");
4907 Opc = AArch64::STR_ZXI;
4912 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4913 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4914 Opc = AArch64::ST1Threev1d;
4919 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4920 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4921 Opc = AArch64::ST1Fourv1d;
4923 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4924 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4925 Opc = AArch64::ST1Twov2d;
4927 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
4928 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4930 "Unexpected register store without SVE store instructions");
4931 Opc = AArch64::STR_ZZXI;
4936 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4937 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4938 Opc = AArch64::ST1Threev2d;
4940 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4942 "Unexpected register store without SVE store instructions");
4943 Opc = AArch64::STR_ZZZXI;
4948 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4949 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4950 Opc = AArch64::ST1Fourv2d;
4952 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
4953 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4955 "Unexpected register store without SVE store instructions");
4956 Opc = AArch64::STR_ZZZZXI;
4961 assert(Opc &&
"Unknown register class");
4972 MI.addMemOperand(MMO);
4979 Register DestReg,
unsigned SubIdx0,
4980 unsigned SubIdx1,
int FI,
4984 bool IsUndef =
true;
4986 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
4988 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5017 switch (
TRI->getSpillSize(*RC)) {
5019 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5020 Opc = AArch64::LDRBui;
5023 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5024 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5025 Opc = AArch64::LDRHui;
5026 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5028 "Unexpected register load without SVE load instructions");
5031 Opc = AArch64::LDR_PXI;
5037 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5038 Opc = AArch64::LDRWui;
5042 assert(DestReg != AArch64::WSP);
5043 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5044 Opc = AArch64::LDRSui;
5045 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5046 Opc = AArch64::LDR_PPXI;
5051 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5052 Opc = AArch64::LDRXui;
5056 assert(DestReg != AArch64::SP);
5057 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5058 Opc = AArch64::LDRDui;
5059 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5061 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5062 AArch64::subo32, FI, MMO);
5067 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5068 Opc = AArch64::LDRQui;
5069 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5070 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5071 Opc = AArch64::LD1Twov1d;
5073 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5075 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5076 AArch64::subo64, FI, MMO);
5078 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5080 "Unexpected register load without SVE load instructions");
5081 Opc = AArch64::LDR_ZXI;
5086 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5087 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5088 Opc = AArch64::LD1Threev1d;
5093 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5094 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5095 Opc = AArch64::LD1Fourv1d;
5097 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5098 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5099 Opc = AArch64::LD1Twov2d;
5101 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5102 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5104 "Unexpected register load without SVE load instructions");
5105 Opc = AArch64::LDR_ZZXI;
5110 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5111 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5112 Opc = AArch64::LD1Threev2d;
5114 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5116 "Unexpected register load without SVE load instructions");
5117 Opc = AArch64::LDR_ZZZXI;
5122 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5123 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5124 Opc = AArch64::LD1Fourv2d;
5126 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5127 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5129 "Unexpected register load without SVE load instructions");
5130 Opc = AArch64::LDR_ZZZZXI;
5136 assert(Opc &&
"Unknown register class");
5146 MI.addMemOperand(MMO);
5157 UseMI.getIterator()),
5159 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5160 I.readsRegister(AArch64::NZCV, TRI);
5169 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5176 ByteSized =
Offset.getFixed();
5177 VGSized =
Offset.getScalable() / 2;
5185 int64_t &NumDataVectors) {
5189 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5191 NumBytes =
Offset.getFixed();
5193 NumPredicateVectors =
Offset.getScalable() / 2;
5198 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5199 NumPredicateVectors > 62) {
5200 NumDataVectors = NumPredicateVectors / 8;
5201 NumPredicateVectors -= NumDataVectors * 8;
5208 int NumVGScaledBytes,
unsigned VG,
5215 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5216 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5219 if (NumVGScaledBytes) {
5220 Expr.
push_back((uint8_t)dwarf::DW_OP_consts);
5223 Expr.
push_back((uint8_t)dwarf::DW_OP_bregx);
5227 Expr.
push_back((uint8_t)dwarf::DW_OP_mul);
5228 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5230 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5231 << std::abs(NumVGScaledBytes) <<
" * VG";
5240 int64_t NumBytes, NumVGScaledBytes;
5243 std::string CommentBuffer;
5246 if (Reg == AArch64::SP)
5248 else if (Reg == AArch64::FP)
5255 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5256 Expr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
5259 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5263 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
5272 unsigned FrameReg,
unsigned Reg,
5274 bool LastAdjustmentWasScalable) {
5275 if (
Offset.getScalable())
5278 if (FrameReg == Reg && !LastAdjustmentWasScalable)
5281 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5288 int64_t NumBytes, NumVGScaledBytes;
5290 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
5292 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5295 if (!NumVGScaledBytes)
5298 std::string CommentBuffer;
5305 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5309 CfaExpr.
push_back(dwarf::DW_CFA_expression);
5325 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
5328 bool *HasWinCFI,
bool EmitCFAOffset,
5331 unsigned MaxEncoding, ShiftSize;
5333 case AArch64::ADDXri:
5334 case AArch64::ADDSXri:
5335 case AArch64::SUBXri:
5336 case AArch64::SUBSXri:
5337 MaxEncoding = 0xfff;
5340 case AArch64::ADDVL_XXI:
5341 case AArch64::ADDPL_XXI:
5342 case AArch64::ADDSVL_XXI:
5343 case AArch64::ADDSPL_XXI:
5358 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
5360 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
5374 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
5376 if (TmpReg == AArch64::XZR)
5378 &AArch64::GPR64RegClass);
5380 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
5381 unsigned LocalShiftSize = 0;
5382 if (ThisVal > MaxEncoding) {
5383 ThisVal = ThisVal >> ShiftSize;
5384 LocalShiftSize = ShiftSize;
5386 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
5387 "Encoding cannot handle value that big");
5389 Offset -= ThisVal << LocalShiftSize;
5394 .
addImm(Sign * (
int)ThisVal);
5404 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
5405 CFAOffset += Change;
5407 CFAOffset -= Change;
5408 if (EmitCFAOffset && DestReg == TmpReg) {
5421 assert(Sign == 1 &&
"SEH directives should always have a positive sign");
5422 int Imm = (int)(ThisVal << LocalShiftSize);
5423 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
5424 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
5433 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
5434 "emit a single SEH directive");
5435 }
else if (DestReg == AArch64::SP) {
5438 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
5451 unsigned DestReg,
unsigned SrcReg,
5454 bool NeedsWinCFI,
bool *HasWinCFI,
5456 unsigned FrameReg) {
5463 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
5465 int64_t Bytes, NumPredicateVectors, NumDataVectors;
5467 Offset, Bytes, NumPredicateVectors, NumDataVectors);
5470 if (Bytes || (!
Offset && SrcReg != DestReg)) {
5471 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
5472 "SP increment/decrement not 8-byte aligned");
5473 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
5476 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
5479 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
5481 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
5488 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
5489 "SetNZCV not supported with SVE vectors");
5490 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
5491 "WinCFI not supported with SVE vectors");
5493 if (NumDataVectors) {
5495 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
5496 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5497 CFAOffset, FrameReg);
5502 if (NumPredicateVectors) {
5503 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
5505 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
5506 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5507 CFAOffset, FrameReg);
5525 if (
MI.isFullCopy()) {
5528 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
5532 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
5537 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
5565 if (
MI.isCopy() && Ops.
size() == 1 &&
5567 (Ops[0] == 0 || Ops[0] == 1)) {
5568 bool IsSpill = Ops[0] == 0;
5569 bool IsFill = !IsSpill;
5581 :
TRI.getMinimalPhysRegClass(Reg);
5587 "Mismatched register size in non subreg COPY");
5594 return &*--InsertPt;
5606 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
5609 "Unexpected subreg on physical register");
5611 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
5613 return &*--InsertPt;
5631 case AArch64::sub_32:
5632 FillRC = &AArch64::GPR32RegClass;
5635 FillRC = &AArch64::FPR32RegClass;
5638 FillRC = &AArch64::FPR64RegClass;
5644 TRI.getRegSizeInBits(*FillRC) &&
5645 "Mismatched regclass size on folded subreg COPY");
5664 bool *OutUseUnscaledOp,
5665 unsigned *OutUnscaledOp,
5666 int64_t *EmittableOffset) {
5668 if (EmittableOffset)
5669 *EmittableOffset = 0;
5670 if (OutUseUnscaledOp)
5671 *OutUseUnscaledOp =
false;
5677 switch (
MI.getOpcode()) {
5680 case AArch64::LD1Rv1d:
5681 case AArch64::LD1Rv2s:
5682 case AArch64::LD1Rv2d:
5683 case AArch64::LD1Rv4h:
5684 case AArch64::LD1Rv4s:
5685 case AArch64::LD1Rv8b:
5686 case AArch64::LD1Rv8h:
5687 case AArch64::LD1Rv16b:
5688 case AArch64::LD1Twov2d:
5689 case AArch64::LD1Threev2d:
5690 case AArch64::LD1Fourv2d:
5691 case AArch64::LD1Twov1d:
5692 case AArch64::LD1Threev1d:
5693 case AArch64::LD1Fourv1d:
5694 case AArch64::ST1Twov2d:
5695 case AArch64::ST1Threev2d:
5696 case AArch64::ST1Fourv2d:
5697 case AArch64::ST1Twov1d:
5698 case AArch64::ST1Threev1d:
5699 case AArch64::ST1Fourv1d:
5700 case AArch64::ST1i8:
5701 case AArch64::ST1i16:
5702 case AArch64::ST1i32:
5703 case AArch64::ST1i64:
5705 case AArch64::IRGstack:
5706 case AArch64::STGloop:
5707 case AArch64::STZGloop:
5712 TypeSize ScaleValue(0U,
false), Width(0U,
false);
5713 int64_t MinOff, MaxOff;
5719 bool IsMulVL = ScaleValue.isScalable();
5720 unsigned Scale = ScaleValue.getKnownMinValue();
5730 std::optional<unsigned> UnscaledOp =
5732 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
5733 if (useUnscaledOp &&
5738 Scale = ScaleValue.getKnownMinValue();
5739 assert(IsMulVL == ScaleValue.isScalable() &&
5740 "Unscaled opcode has different value for scalable");
5742 int64_t Remainder =
Offset % Scale;
5743 assert(!(Remainder && useUnscaledOp) &&
5744 "Cannot have remainder when using unscaled op");
5746 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
5747 int64_t NewOffset =
Offset / Scale;
5748 if (MinOff <= NewOffset && NewOffset <= MaxOff)
5751 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
5755 if (EmittableOffset)
5756 *EmittableOffset = NewOffset;
5757 if (OutUseUnscaledOp)
5758 *OutUseUnscaledOp = useUnscaledOp;
5759 if (OutUnscaledOp && UnscaledOp)
5760 *OutUnscaledOp = *UnscaledOp;
5773 unsigned Opcode =
MI.getOpcode();
5774 unsigned ImmIdx = FrameRegIdx + 1;
5776 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
5781 MI.eraseFromParent();
5787 unsigned UnscaledOp;
5790 &UnscaledOp, &NewOffset);
5794 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
5796 MI.setDesc(
TII->get(UnscaledOp));
5798 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
5821 case AArch64::ADDSWrr:
5822 case AArch64::ADDSWri:
5823 case AArch64::ADDSXrr:
5824 case AArch64::ADDSXri:
5825 case AArch64::SUBSWrr:
5826 case AArch64::SUBSXrr:
5828 case AArch64::SUBSWri:
5829 case AArch64::SUBSXri:
5840 case AArch64::ADDWrr:
5841 case AArch64::ADDWri:
5842 case AArch64::SUBWrr:
5843 case AArch64::ADDSWrr:
5844 case AArch64::ADDSWri:
5845 case AArch64::SUBSWrr:
5847 case AArch64::SUBWri:
5848 case AArch64::SUBSWri:
5859 case AArch64::ADDXrr:
5860 case AArch64::ADDXri:
5861 case AArch64::SUBXrr:
5862 case AArch64::ADDSXrr:
5863 case AArch64::ADDSXri:
5864 case AArch64::SUBSXrr:
5866 case AArch64::SUBXri:
5867 case AArch64::SUBSXri:
5868 case AArch64::ADDv8i8:
5869 case AArch64::ADDv16i8:
5870 case AArch64::ADDv4i16:
5871 case AArch64::ADDv8i16:
5872 case AArch64::ADDv2i32:
5873 case AArch64::ADDv4i32:
5874 case AArch64::SUBv8i8:
5875 case AArch64::SUBv16i8:
5876 case AArch64::SUBv4i16:
5877 case AArch64::SUBv8i16:
5878 case AArch64::SUBv2i32:
5879 case AArch64::SUBv4i32:
5892 case AArch64::FADDHrr:
5893 case AArch64::FADDSrr:
5894 case AArch64::FADDDrr:
5895 case AArch64::FADDv4f16:
5896 case AArch64::FADDv8f16:
5897 case AArch64::FADDv2f32:
5898 case AArch64::FADDv2f64:
5899 case AArch64::FADDv4f32:
5900 case AArch64::FSUBHrr:
5901 case AArch64::FSUBSrr:
5902 case AArch64::FSUBDrr:
5903 case AArch64::FSUBv4f16:
5904 case AArch64::FSUBv8f16:
5905 case AArch64::FSUBv2f32:
5906 case AArch64::FSUBv2f64:
5907 case AArch64::FSUBv4f32:
5911 return Options.UnsafeFPMath ||
5928 unsigned CombineOpc,
unsigned ZeroReg = 0,
5929 bool CheckZeroReg =
false) {
5936 if (!
MI ||
MI->getParent() != &
MBB || (
unsigned)
MI->getOpcode() != CombineOpc)
5939 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
5943 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
5944 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
5945 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
5947 if (
MI->getOperand(3).getReg() != ZeroReg)
5952 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
5961 unsigned MulOpc,
unsigned ZeroReg) {
5977 bool Invert)
const {
5983 case AArch64::FADDHrr:
5984 case AArch64::FADDSrr:
5985 case AArch64::FADDDrr:
5986 case AArch64::FMULHrr:
5987 case AArch64::FMULSrr:
5988 case AArch64::FMULDrr:
5989 case AArch64::FMULX16:
5990 case AArch64::FMULX32:
5991 case AArch64::FMULX64:
5993 case AArch64::FADDv4f16:
5994 case AArch64::FADDv8f16:
5995 case AArch64::FADDv2f32:
5996 case AArch64::FADDv4f32:
5997 case AArch64::FADDv2f64:
5998 case AArch64::FMULv4f16:
5999 case AArch64::FMULv8f16:
6000 case AArch64::FMULv2f32:
6001 case AArch64::FMULv4f32:
6002 case AArch64::FMULv2f64:
6003 case AArch64::FMULXv4f16:
6004 case AArch64::FMULXv8f16:
6005 case AArch64::FMULXv2f32:
6006 case AArch64::FMULXv4f32:
6007 case AArch64::FMULXv2f64:
6011 case AArch64::FADD_ZZZ_H:
6012 case AArch64::FADD_ZZZ_S:
6013 case AArch64::FADD_ZZZ_D:
6014 case AArch64::FMUL_ZZZ_H:
6015 case AArch64::FMUL_ZZZ_S:
6016 case AArch64::FMUL_ZZZ_D:
6028 case AArch64::ADDWrr:
6029 case AArch64::ADDXrr:
6030 case AArch64::ANDWrr:
6031 case AArch64::ANDXrr:
6032 case AArch64::ORRWrr:
6033 case AArch64::ORRXrr:
6034 case AArch64::EORWrr:
6035 case AArch64::EORXrr:
6036 case AArch64::EONWrr:
6037 case AArch64::EONXrr:
6041 case AArch64::ADDv8i8:
6042 case AArch64::ADDv16i8:
6043 case AArch64::ADDv4i16:
6044 case AArch64::ADDv8i16:
6045 case AArch64::ADDv2i32:
6046 case AArch64::ADDv4i32:
6047 case AArch64::ADDv1i64:
6048 case AArch64::ADDv2i64:
6049 case AArch64::MULv8i8:
6050 case AArch64::MULv16i8:
6051 case AArch64::MULv4i16:
6052 case AArch64::MULv8i16:
6053 case AArch64::MULv2i32:
6054 case AArch64::MULv4i32:
6055 case AArch64::ANDv8i8:
6056 case AArch64::ANDv16i8:
6057 case AArch64::ORRv8i8:
6058 case AArch64::ORRv16i8:
6059 case AArch64::EORv8i8:
6060 case AArch64::EORv16i8:
6062 case AArch64::ADD_ZZZ_B:
6063 case AArch64::ADD_ZZZ_H:
6064 case AArch64::ADD_ZZZ_S:
6065 case AArch64::ADD_ZZZ_D:
6066 case AArch64::MUL_ZZZ_B:
6067 case AArch64::MUL_ZZZ_H:
6068 case AArch64::MUL_ZZZ_S:
6069 case AArch64::MUL_ZZZ_D:
6070 case AArch64::AND_ZZZ:
6071 case AArch64::ORR_ZZZ:
6072 case AArch64::EOR_ZZZ:
6103 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6111 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6123 case AArch64::ADDWrr:
6125 "ADDWrr does not have register operands");
6126 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6127 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6129 case AArch64::ADDXrr:
6130 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6131 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6133 case AArch64::SUBWrr:
6134 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6135 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6137 case AArch64::SUBXrr:
6138 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6139 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6141 case AArch64::ADDWri:
6142 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6144 case AArch64::ADDXri:
6145 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6147 case AArch64::SUBWri:
6148 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6150 case AArch64::SUBXri:
6151 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6153 case AArch64::ADDv8i8:
6154 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6155 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6157 case AArch64::ADDv16i8:
6158 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6159 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6161 case AArch64::ADDv4i16:
6162 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6163 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6164 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6165 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6167 case AArch64::ADDv8i16:
6168 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6169 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6170 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6171 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6173 case AArch64::ADDv2i32:
6174 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6175 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6176 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6177 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6179 case AArch64::ADDv4i32:
6180 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6181 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6182 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6183 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6185 case AArch64::SUBv8i8:
6186 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6187 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6189 case AArch64::SUBv16i8:
6190 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6191 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6193 case AArch64::SUBv4i16:
6194 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6195 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
6196 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
6197 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
6199 case AArch64::SUBv8i16:
6200 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
6201 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
6202 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
6203 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
6205 case AArch64::SUBv2i32:
6206 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
6207 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
6208 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
6209 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
6211 case AArch64::SUBv4i32:
6212 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
6213 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
6214 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
6215 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
6232 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
6244 assert(
false &&
"Unsupported FP instruction in combiner\n");
6246 case AArch64::FADDHrr:
6248 "FADDHrr does not have register operands");
6250 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
6251 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
6253 case AArch64::FADDSrr:
6255 "FADDSrr does not have register operands");
6257 Found |=
Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
6258 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
6260 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
6261 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
6263 case AArch64::FADDDrr:
6264 Found |=
Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
6265 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
6267 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
6268 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
6270 case AArch64::FADDv4f16:
6271 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
6272 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
6274 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
6275 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
6277 case AArch64::FADDv8f16:
6278 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
6279 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
6281 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
6282 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
6284 case AArch64::FADDv2f32:
6285 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
6286 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
6288 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
6289 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
6291 case AArch64::FADDv2f64:
6292 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
6293 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
6295 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
6296 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
6298 case AArch64::FADDv4f32:
6299 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
6300 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
6302 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
6303 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
6305 case AArch64::FSUBHrr:
6306 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
6307 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
6308 Found |=
Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
6310 case AArch64::FSUBSrr:
6311 Found =
Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
6313 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
6314 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
6316 Found |=
Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
6318 case AArch64::FSUBDrr:
6319 Found =
Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
6321 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
6322 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
6324 Found |=
Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
6326 case AArch64::FSUBv4f16:
6327 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
6328 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
6330 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
6331 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
6333 case AArch64::FSUBv8f16:
6334 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
6335 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
6337 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
6338 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
6340 case AArch64::FSUBv2f32:
6341 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
6342 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
6344 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
6345 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
6347 case AArch64::FSUBv2f64:
6348 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
6349 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
6351 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
6352 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
6354 case AArch64::FSUBv4f32:
6355 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
6356 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
6358 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
6359 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
6370 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
6377 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
6378 MI->getOperand(1).getReg().isVirtual())
6379 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
6380 if (
MI &&
MI->getOpcode() == Opcode) {
6392 case AArch64::FMULv2f32:
6393 Found =
Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
6394 Found |=
Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
6396 case AArch64::FMULv2f64:
6397 Found =
Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
6398 Found |=
Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
6400 case AArch64::FMULv4f16:
6401 Found =
Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
6402 Found |=
Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
6404 case AArch64::FMULv4f32:
6405 Found =
Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
6406 Found |=
Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
6408 case AArch64::FMULv8f16:
6409 Found =
Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
6410 Found |=
Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
6423 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
6426 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
6427 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
6441 case AArch64::FNEGDr:
6443 case AArch64::FNEGSr:
6575 case AArch64::SUBWrr:
6576 case AArch64::SUBSWrr:
6577 case AArch64::SUBXrr:
6578 case AArch64::SUBSXrr:
6620 bool DoRegPressureReduce)
const {
6637 DoRegPressureReduce);
6666 const Register *ReplacedAddend =
nullptr) {
6667 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6669 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
6672 Register SrcReg0 = MUL->getOperand(1).getReg();
6673 bool Src0IsKill = MUL->getOperand(1).isKill();
6674 Register SrcReg1 = MUL->getOperand(2).getReg();
6675 bool Src1IsKill = MUL->getOperand(2).isKill();
6679 if (ReplacedAddend) {
6681 SrcReg2 = *ReplacedAddend;
6689 MRI.constrainRegClass(ResultReg, RC);
6691 MRI.constrainRegClass(SrcReg0, RC);
6693 MRI.constrainRegClass(SrcReg1, RC);
6695 MRI.constrainRegClass(SrcReg2, RC);
6698 if (kind == FMAInstKind::Default)
6703 else if (kind == FMAInstKind::Indexed)
6708 .
addImm(MUL->getOperand(3).getImm());
6709 else if (kind == FMAInstKind::Accumulator)
6715 assert(
false &&
"Invalid FMA instruction kind \n");
6729 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6730 Opc = AArch64::FNMADDSrrr;
6731 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
6732 Opc = AArch64::FNMADDDrrr;
6744 MRI.constrainRegClass(ResultReg, RC);
6746 MRI.constrainRegClass(SrcReg0, RC);
6748 MRI.constrainRegClass(SrcReg1, RC);
6750 MRI.constrainRegClass(SrcReg2, RC);
6766 unsigned IdxDupOp,
unsigned MulOpc,
6768 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
6769 "Invalid index of FMUL operand");
6777 if (Dup->
getOpcode() == TargetOpcode::COPY)
6781 MRI.clearKillFlags(DupSrcReg);
6782 MRI.constrainRegClass(DupSrcReg, RC);
6786 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
6810 FMAInstKind::Accumulator);
6827 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6842 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6844 FMAInstKind::Accumulator, &NewVR);
6856 FMAInstKind::Indexed);
6869 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6872 FMAInstKind::Indexed, &NewVR);
6897 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
6899 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6903 Register SrcReg0 = MUL->getOperand(1).getReg();
6904 bool Src0IsKill = MUL->getOperand(1).isKill();
6905 Register SrcReg1 = MUL->getOperand(2).getReg();
6906 bool Src1IsKill = MUL->getOperand(2).isKill();
6909 MRI.constrainRegClass(ResultReg, RC);
6911 MRI.constrainRegClass(SrcReg0, RC);
6913 MRI.constrainRegClass(SrcReg1, RC);
6915 MRI.constrainRegClass(VR, RC);
6937 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
6938 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
6948 Register NewVR =
MRI.createVirtualRegister(
MRI.getRegClass(RegA));
6951 if (Opcode == AArch64::SUBSWrr)
6952 Opcode = AArch64::SUBWrr;
6953 else if (Opcode == AArch64::SUBSXrr)
6954 Opcode = AArch64::SUBXrr;
6956 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
6957 "Unexpected instruction opcode.");
6974 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7001 DelInstrs, InstrIdxForVirtReg);
7007 InstrIdxForVirtReg);
7013 InstrIdxForVirtReg);
7022 Opc = AArch64::MADDWrrr;
7023 RC = &AArch64::GPR32RegClass;
7025 Opc = AArch64::MADDXrrr;
7026 RC = &AArch64::GPR64RegClass;
7037 Opc = AArch64::MADDWrrr;
7038 RC = &AArch64::GPR32RegClass;
7040 Opc = AArch64::MADDXrrr;
7041 RC = &AArch64::GPR64RegClass;
7053 unsigned BitSize, OrrOpc, ZeroReg;
7055 OrrOpc = AArch64::ORRWri;
7056 OrrRC = &AArch64::GPR32spRegClass;
7058 ZeroReg = AArch64::WZR;
7059 Opc = AArch64::MADDWrrr;
7060 RC = &AArch64::GPR32RegClass;
7062 OrrOpc = AArch64::ORRXri;
7063 OrrRC = &AArch64::GPR64spRegClass;
7065 ZeroReg = AArch64::XZR;
7066 Opc = AArch64::MADDXrrr;
7067 RC = &AArch64::GPR64RegClass;
7069 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7080 if (
Insn.size() != 1)
7082 auto MovI =
Insn.begin();
7085 if (MovI->Opcode == OrrOpc)
7091 assert((MovI->Opcode == AArch64::MOVNWi ||
7092 MovI->Opcode == AArch64::MOVZWi) &&
7095 assert((MovI->Opcode == AArch64::MOVNXi ||
7096 MovI->Opcode == AArch64::MOVZXi) &&
7103 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7104 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7115 unsigned SubOpc, ZeroReg;
7117 SubOpc = AArch64::SUBWrr;
7118 SubRC = &AArch64::GPR32spRegClass;
7119 ZeroReg = AArch64::WZR;
7120 Opc = AArch64::MADDWrrr;
7121 RC = &AArch64::GPR32RegClass;
7123 SubOpc = AArch64::SUBXrr;
7124 SubRC = &AArch64::GPR64spRegClass;
7125 ZeroReg = AArch64::XZR;
7126 Opc = AArch64::MADDXrrr;
7127 RC = &AArch64::GPR64RegClass;
7129 Register NewVR =
MRI.createVirtualRegister(SubRC);
7136 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7137 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7147 Opc = AArch64::MSUBWrrr;
7148 RC = &AArch64::GPR32RegClass;
7150 Opc = AArch64::MSUBXrrr;
7151 RC = &AArch64::GPR64RegClass;
7163 unsigned BitSize, OrrOpc, ZeroReg;
7165 OrrOpc = AArch64::ORRWri;
7166 OrrRC = &AArch64::GPR32spRegClass;
7168 ZeroReg = AArch64::WZR;
7169 Opc = AArch64::MADDWrrr;
7170 RC = &AArch64::GPR32RegClass;
7172 OrrOpc = AArch64::ORRXri;
7173 OrrRC = &AArch64::GPR64spRegClass;
7175 ZeroReg = AArch64::XZR;
7176 Opc = AArch64::MADDXrrr;
7177 RC = &AArch64::GPR64RegClass;
7179 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7189 if (
Insn.size() != 1)
7191 auto MovI =
Insn.begin();
7194 if (MovI->Opcode == OrrOpc)
7200 assert((MovI->Opcode == AArch64::MOVNWi ||
7201 MovI->Opcode == AArch64::MOVZWi) &&
7204 assert((MovI->Opcode == AArch64::MOVNXi ||
7205 MovI->Opcode == AArch64::MOVZXi) &&
7212 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7213 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7218 Opc = AArch64::MLAv8i8;
7219 RC = &AArch64::FPR64RegClass;
7223 Opc = AArch64::MLAv8i8;
7224 RC = &AArch64::FPR64RegClass;
7228 Opc = AArch64::MLAv16i8;
7229 RC = &AArch64::FPR128RegClass;
7233 Opc = AArch64::MLAv16i8;
7234 RC = &AArch64::FPR128RegClass;
7238 Opc = AArch64::MLAv4i16;
7239 RC = &AArch64::FPR64RegClass;
7243 Opc = AArch64::MLAv4i16;
7244 RC = &AArch64::FPR64RegClass;
7248 Opc = AArch64::MLAv8i16;
7249 RC = &AArch64::FPR128RegClass;
7253 Opc = AArch64::MLAv8i16;
7254 RC = &AArch64::FPR128RegClass;
7258 Opc = AArch64::MLAv2i32;
7259 RC = &AArch64::FPR64RegClass;
7263 Opc = AArch64::MLAv2i32;
7264 RC = &AArch64::FPR64RegClass;
7268 Opc = AArch64::MLAv4i32;
7269 RC = &AArch64::FPR128RegClass;
7273 Opc = AArch64::MLAv4i32;
7274 RC = &AArch64::FPR128RegClass;
7279 Opc = AArch64::MLAv8i8;
7280 RC = &AArch64::FPR64RegClass;
7282 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
7286 Opc = AArch64::MLSv8i8;
7287 RC = &AArch64::FPR64RegClass;
7291 Opc = AArch64::MLAv16i8;
7292 RC = &AArch64::FPR128RegClass;
7294 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
7298 Opc = AArch64::MLSv16i8;
7299 RC = &AArch64::FPR128RegClass;
7303 Opc = AArch64::MLAv4i16;
7304 RC = &AArch64::FPR64RegClass;
7306 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7310 Opc = AArch64::MLSv4i16;
7311 RC = &AArch64::FPR64RegClass;
7315 Opc = AArch64::MLAv8i16;
7316 RC = &AArch64::FPR128RegClass;
7318 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7322 Opc = AArch64::MLSv8i16;
7323 RC = &AArch64::FPR128RegClass;
7327 Opc = AArch64::MLAv2i32;
7328 RC = &AArch64::FPR64RegClass;
7330 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7334 Opc = AArch64::MLSv2i32;
7335 RC = &AArch64::FPR64RegClass;
7339 Opc = AArch64::MLAv4i32;
7340 RC = &AArch64::FPR128RegClass;
7342 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7346 Opc = AArch64::MLSv4i32;
7347 RC = &AArch64::FPR128RegClass;
7352 Opc = AArch64::MLAv4i16_indexed;
7353 RC = &AArch64::FPR64RegClass;
7357 Opc = AArch64::MLAv4i16_indexed;
7358 RC = &AArch64::FPR64RegClass;
7362 Opc = AArch64::MLAv8i16_indexed;
7363 RC = &AArch64::FPR128RegClass;
7367 Opc = AArch64::MLAv8i16_indexed;
7368 RC = &AArch64::FPR128RegClass;
7372 Opc = AArch64::MLAv2i32_indexed;
7373 RC = &AArch64::FPR64RegClass;
7377 Opc = AArch64::MLAv2i32_indexed;
7378 RC = &AArch64::FPR64RegClass;
7382 Opc = AArch64::MLAv4i32_indexed;
7383 RC = &AArch64::FPR128RegClass;
7387 Opc = AArch64::MLAv4i32_indexed;
7388 RC = &AArch64::FPR128RegClass;
7393 Opc = AArch64::MLAv4i16_indexed;
7394 RC = &AArch64::FPR64RegClass;
7396 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7400 Opc = AArch64::MLSv4i16_indexed;
7401 RC = &AArch64::FPR64RegClass;
7405 Opc = AArch64::MLAv8i16_indexed;
7406 RC = &AArch64::FPR128RegClass;
7408 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7412 Opc = AArch64::MLSv8i16_indexed;
7413 RC = &AArch64::FPR128RegClass;
7417 Opc = AArch64::MLAv2i32_indexed;
7418 RC = &AArch64::FPR64RegClass;
7420 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7424 Opc = AArch64::MLSv2i32_indexed;
7425 RC = &AArch64::FPR64RegClass;
7429 Opc = AArch64::MLAv4i32_indexed;
7430 RC = &AArch64::FPR128RegClass;
7432 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7436 Opc = AArch64::MLSv4i32_indexed;
7437 RC = &AArch64::FPR128RegClass;
7443 Opc = AArch64::FMADDHrrr;
7444 RC = &AArch64::FPR16RegClass;
7448 Opc = AArch64::FMADDSrrr;
7449 RC = &AArch64::FPR32RegClass;
7453 Opc = AArch64::FMADDDrrr;
7454 RC = &AArch64::FPR64RegClass;
7459 Opc = AArch64::FMADDHrrr;
7460 RC = &AArch64::FPR16RegClass;
7464 Opc = AArch64::FMADDSrrr;
7465 RC = &AArch64::FPR32RegClass;
7469 Opc = AArch64::FMADDDrrr;
7470 RC = &AArch64::FPR64RegClass;
7475 Opc = AArch64::FMLAv1i32_indexed;
7476 RC = &AArch64::FPR32RegClass;
7478 FMAInstKind::Indexed);
7481 Opc = AArch64::FMLAv1i32_indexed;
7482 RC = &AArch64::FPR32RegClass;
7484 FMAInstKind::Indexed);
7488 Opc = AArch64::FMLAv1i64_indexed;
7489 RC = &AArch64::FPR64RegClass;
7491 FMAInstKind::Indexed);
7494 Opc = AArch64::FMLAv1i64_indexed;
7495 RC = &AArch64::FPR64RegClass;
7497 FMAInstKind::Indexed);
7501 RC = &AArch64::FPR64RegClass;
7502 Opc = AArch64::FMLAv4i16_indexed;
7504 FMAInstKind::Indexed);
7507 RC = &AArch64::FPR64RegClass;
7508 Opc = AArch64::FMLAv4f16;
7510 FMAInstKind::Accumulator);
7513 RC = &AArch64::FPR64RegClass;
7514 Opc = AArch64::FMLAv4i16_indexed;
7516 FMAInstKind::Indexed);
7519 RC = &AArch64::FPR64RegClass;
7520 Opc = AArch64::FMLAv4f16;
7522 FMAInstKind::Accumulator);
7527 RC = &AArch64::FPR64RegClass;
7529 Opc = AArch64::FMLAv2i32_indexed;
7531 FMAInstKind::Indexed);
7533 Opc = AArch64::FMLAv2f32;
7535 FMAInstKind::Accumulator);
7540 RC = &AArch64::FPR64RegClass;
7542 Opc = AArch64::FMLAv2i32_indexed;
7544 FMAInstKind::Indexed);
7546 Opc = AArch64::FMLAv2f32;
7548 FMAInstKind::Accumulator);
7553 RC = &AArch64::FPR128RegClass;
7554 Opc = AArch64::FMLAv8i16_indexed;
7556 FMAInstKind::Indexed);
7559 RC = &AArch64::FPR128RegClass;
7560 Opc = AArch64::FMLAv8f16;
7562 FMAInstKind::Accumulator);
7565 RC = &AArch64::FPR128RegClass;
7566 Opc = AArch64::FMLAv8i16_indexed;
7568 FMAInstKind::Indexed);
7571 RC = &AArch64::FPR128RegClass;
7572 Opc = AArch64::FMLAv8f16;
7574 FMAInstKind::Accumulator);
7579 RC = &AArch64::FPR128RegClass;
7581 Opc = AArch64::FMLAv2i64_indexed;
7583 FMAInstKind::Indexed);
7585 Opc = AArch64::FMLAv2f64;
7587 FMAInstKind::Accumulator);
7592 RC = &AArch64::FPR128RegClass;
7594 Opc = AArch64::FMLAv2i64_indexed;
7596 FMAInstKind::Indexed);
7598 Opc = AArch64::FMLAv2f64;
7600 FMAInstKind::Accumulator);
7606 RC = &AArch64::FPR128RegClass;
7608 Opc = AArch64::FMLAv4i32_indexed;
7610 FMAInstKind::Indexed);
7612 Opc = AArch64::FMLAv4f32;
7614 FMAInstKind::Accumulator);
7620 RC = &AArch64::FPR128RegClass;
7622 Opc = AArch64::FMLAv4i32_indexed;
7624 FMAInstKind::Indexed);
7626 Opc = AArch64::FMLAv4f32;
7628 FMAInstKind::Accumulator);
7633 Opc = AArch64::FNMSUBHrrr;
7634 RC = &AArch64::FPR16RegClass;
7638 Opc = AArch64::FNMSUBSrrr;
7639 RC = &AArch64::FPR32RegClass;
7643 Opc = AArch64::FNMSUBDrrr;
7644 RC = &AArch64::FPR64RegClass;
7649 Opc = AArch64::FNMADDHrrr;
7650 RC = &AArch64::FPR16RegClass;
7654 Opc = AArch64::FNMADDSrrr;
7655 RC = &AArch64::FPR32RegClass;
7659 Opc = AArch64::FNMADDDrrr;
7660 RC = &AArch64::FPR64RegClass;
7665 Opc = AArch64::FMSUBHrrr;
7666 RC = &AArch64::FPR16RegClass;
7670 Opc = AArch64::FMSUBSrrr;
7671 RC = &AArch64::FPR32RegClass;
7675 Opc = AArch64::FMSUBDrrr;
7676 RC = &AArch64::FPR64RegClass;
7681 Opc = AArch64::FMLSv1i32_indexed;
7682 RC = &AArch64::FPR32RegClass;
7684 FMAInstKind::Indexed);
7688 Opc = AArch64::FMLSv1i64_indexed;
7689 RC = &AArch64::FPR64RegClass;
7691 FMAInstKind::Indexed);
7696 RC = &AArch64::FPR64RegClass;
7702 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7704 Opc = AArch64::FMLAv4f16;
7706 FMAInstKind::Accumulator, &NewVR);
7708 Opc = AArch64::FMLAv4i16_indexed;
7710 FMAInstKind::Indexed, &NewVR);
7715 RC = &AArch64::FPR64RegClass;
7716 Opc = AArch64::FMLSv4f16;
7718 FMAInstKind::Accumulator);
7721 RC = &AArch64::FPR64RegClass;
7722 Opc = AArch64::FMLSv4i16_indexed;
7724 FMAInstKind::Indexed);
7729 RC = &AArch64::FPR64RegClass;
7731 Opc = AArch64::FMLSv2i32_indexed;
7733 FMAInstKind::Indexed);
7735 Opc = AArch64::FMLSv2f32;
7737 FMAInstKind::Accumulator);
7743 RC = &AArch64::FPR128RegClass;
7749 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7751 Opc = AArch64::FMLAv8f16;
7753 FMAInstKind::Accumulator, &NewVR);
7755 Opc = AArch64::FMLAv8i16_indexed;
7757 FMAInstKind::Indexed, &NewVR);
7762 RC = &AArch64::FPR128RegClass;
7763 Opc = AArch64::FMLSv8f16;
7765 FMAInstKind::Accumulator);
7768 RC = &AArch64::FPR128RegClass;
7769 Opc = AArch64::FMLSv8i16_indexed;
7771 FMAInstKind::Indexed);
7776 RC = &AArch64::FPR128RegClass;
7778 Opc = AArch64::FMLSv2i64_indexed;
7780 FMAInstKind::Indexed);
7782 Opc = AArch64::FMLSv2f64;
7784 FMAInstKind::Accumulator);
7790 RC = &AArch64::FPR128RegClass;
7792 Opc = AArch64::FMLSv4i32_indexed;
7794 FMAInstKind::Indexed);
7796 Opc = AArch64::FMLSv4f32;
7798 FMAInstKind::Accumulator);
7803 RC = &AArch64::FPR64RegClass;
7809 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7811 Opc = AArch64::FMLAv2i32_indexed;
7813 FMAInstKind::Indexed, &NewVR);
7815 Opc = AArch64::FMLAv2f32;
7817 FMAInstKind::Accumulator, &NewVR);
7823 RC = &AArch64::FPR128RegClass;
7829 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7831 Opc = AArch64::FMLAv4i32_indexed;
7833 FMAInstKind::Indexed, &NewVR);
7835 Opc = AArch64::FMLAv4f32;
7837 FMAInstKind::Accumulator, &NewVR);
7843 RC = &AArch64::FPR128RegClass;
7849 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7851 Opc = AArch64::FMLAv2i64_indexed;
7853 FMAInstKind::Indexed, &NewVR);
7855 Opc = AArch64::FMLAv2f64;
7857 FMAInstKind::Accumulator, &NewVR);
7867 &AArch64::FPR128RegClass,
MRI);
7876 &AArch64::FPR128RegClass,
MRI);
7885 &AArch64::FPR128_loRegClass,
MRI);
7894 &AArch64::FPR128RegClass,
MRI);
7903 &AArch64::FPR128_loRegClass,
MRI);
7922 for (
auto *
MI : InsInstrs)
7923 MI->setFlags(Flags);
7964 bool IsNegativeBranch =
false;
7965 bool IsTestAndBranch =
false;
7966 unsigned TargetBBInMI = 0;
7967 switch (
MI.getOpcode()) {
7976 case AArch64::CBNZW:
7977 case AArch64::CBNZX:
7979 IsNegativeBranch =
true;
7984 IsTestAndBranch =
true;
7986 case AArch64::TBNZW:
7987 case AArch64::TBNZX:
7989 IsNegativeBranch =
true;
7990 IsTestAndBranch =
true;
7996 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
8000 assert(
MI.getParent() &&
"Incomplete machine instruciton\n");
8013 if (!
MRI->hasOneNonDBGUse(CopyVReg))
8015 if (!
MRI->hasOneDef(CopyVReg))
8024 case AArch64::ANDWri:
8025 case AArch64::ANDXri: {
8026 if (IsTestAndBranch)
8030 if (!
MRI->hasOneNonDBGUse(VReg))
8044 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
8050 unsigned Opc = (Imm < 32)
8051 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
8052 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
8065 if (!Is32Bit && Imm < 32)
8067 MI.eraseFromParent();
8071 case AArch64::CSINCWr:
8072 case AArch64::CSINCXr: {
8092 if (IsNegativeBranch)
8095 MI.eraseFromParent();
8101std::pair<unsigned, unsigned>
8104 return std::make_pair(TF & Mask, TF & ~Mask);
8109 using namespace AArch64II;
8111 static const std::pair<unsigned, const char *> TargetFlags[] = {
8112 {MO_PAGE,
"aarch64-page"}, {MO_PAGEOFF,
"aarch64-pageoff"},
8113 {MO_G3,
"aarch64-g3"}, {MO_G2,
"aarch64-g2"},
8114 {MO_G1,
"aarch64-g1"}, {MO_G0,
"aarch64-g0"},
8115 {MO_HI12,
"aarch64-hi12"}};
8121 using namespace AArch64II;
8123 static const std::pair<unsigned, const char *> TargetFlags[] = {
8124 {MO_COFFSTUB,
"aarch64-coffstub"},
8125 {MO_GOT,
"aarch64-got"},
8126 {MO_NC,
"aarch64-nc"},
8127 {MO_S,
"aarch64-s"},
8128 {MO_TLS,
"aarch64-tls"},
8129 {MO_DLLIMPORT,
"aarch64-dllimport"},
8130 {MO_PREL,
"aarch64-prel"},
8131 {MO_TAGGED,
"aarch64-tagged"},
8132 {MO_ARM64EC_CALLMANGLE,
"aarch64-arm64ec-callmangle"},
8139 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8247 for (
unsigned Reg : AArch64::GPR64RegClass) {
8249 Reg != AArch64::LR &&
8250 Reg != AArch64::X16 &&
8251 Reg != AArch64::X17 &&
8252 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
8253 C.isAvailableInsideSeq(
Reg,
TRI))
8284 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
8287std::optional<outliner::OutlinedFunction>
8290 std::vector<outliner::Candidate> &RepeatedSequenceLocs)
const {
8291 unsigned SequenceSize = 0;
8292 for (
auto &
MI : RepeatedSequenceLocs[0])
8295 unsigned NumBytesToCreateFrame = 0;
8305 if (std::adjacent_find(
8306 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
8310 if (outliningCandidatesSigningScopeConsensus(a, b) &&
8311 outliningCandidatesSigningKeyConsensus(a, b) &&
8312 outliningCandidatesV8_3OpsConsensus(a, b)) {
8316 }) != RepeatedSequenceLocs.end()) {
8317 return std::nullopt;
8334 unsigned NumBytesToCheckLRInTCEpilogue = 0;
8335 if (RepeatedSequenceLocs[0]
8337 ->getInfo<AArch64FunctionInfo>()
8338 ->shouldSignReturnAddress(
true)) {
8340 NumBytesToCreateFrame += 8;
8343 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod();
8344 NumBytesToCheckLRInTCEpilogue =
8348 if (isTailCallReturnInst(RepeatedSequenceLocs[0].back()))
8349 SequenceSize += NumBytesToCheckLRInTCEpilogue;
8357 for (
auto &
MI :
C) {
8358 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
8359 switch (
MI.getOpcode()) {
8360 case AArch64::ADDXri:
8361 case AArch64::ADDWri:
8362 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8364 "Expected operand to be immediate");
8366 "Expected operand to be a register");
8370 if (
MI.getOperand(1).getReg() == AArch64::SP)
8371 SPValue +=
MI.getOperand(2).getImm();
8375 case AArch64::SUBXri:
8376 case AArch64::SUBWri:
8377 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8379 "Expected operand to be immediate");
8381 "Expected operand to be a register");
8385 if (
MI.getOperand(1).getReg() == AArch64::SP)
8386 SPValue -=
MI.getOperand(2).getImm();
8403 if (RepeatedSequenceLocs.size() < 2)
8404 return std::nullopt;
8408 unsigned FlagsSetInAll = 0xF;
8412 FlagsSetInAll &=
C.Flags;
8414 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
8417 auto SetCandidateCallInfo =
8418 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
8420 C.setCallInfo(CallID, NumBytesForCall);
8424 NumBytesToCreateFrame += 4;
8427 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
8432 unsigned CFICount = 0;
8433 for (
auto &
I : RepeatedSequenceLocs[0]) {
8434 if (
I.isCFIInstruction())
8444 std::vector<MCCFIInstruction> CFIInstructions =
8445 C.getMF()->getFrameInstructions();
8447 if (CFICount > 0 && CFICount != CFIInstructions.size())
8448 return std::nullopt;
8456 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
8457 !
MI.readsRegister(AArch64::SP, &
TRI))
8463 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
8468 if (
MI.mayLoadOrStore()) {
8471 bool OffsetIsScalable;
8475 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
8476 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
8480 if (OffsetIsScalable)
8488 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
8489 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
8492 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
8493 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
8508 bool AllStackInstrsSafe =
8513 if (RepeatedSequenceLocs[0].back().isTerminator()) {
8515 NumBytesToCreateFrame = 0;
8516 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
8520 else if (LastInstrOpcode == AArch64::BL ||
8521 ((LastInstrOpcode == AArch64::BLR ||
8522 LastInstrOpcode == AArch64::BLRNoIP) &&
8526 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
8534 unsigned NumBytesNoStackCalls = 0;
8535 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
8540 (
C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
8541 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
8550 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
8553 if (LRAvailable && !IsNoReturn) {
8554 NumBytesNoStackCalls += 4;
8556 CandidatesWithoutStackFixups.push_back(
C);
8561 else if (findRegisterToSaveLRTo(
C)) {
8562 NumBytesNoStackCalls += 12;
8564 CandidatesWithoutStackFixups.push_back(
C);
8569 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
8570 NumBytesNoStackCalls += 12;
8572 CandidatesWithoutStackFixups.push_back(
C);
8578 NumBytesNoStackCalls += SequenceSize;
8585 if (!AllStackInstrsSafe ||
8586 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
8587 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
8589 if (RepeatedSequenceLocs.size() < 2)
8590 return std::nullopt;
8639 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8643 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
8644 !findRegisterToSaveLRTo(
C));
8650 if (RepeatedSequenceLocs.size() < 2) {
8651 RepeatedSequenceLocs.clear();
8652 return std::nullopt;
8658 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8662 bool ModStackToSaveLR =
false;
8663 if (std::any_of(FirstCand.
begin(), std::prev(FirstCand.
end()),
8665 ModStackToSaveLR =
true;
8674 ModStackToSaveLR =
true;
8676 if (ModStackToSaveLR) {
8678 if (!AllStackInstrsSafe) {
8679 RepeatedSequenceLocs.clear();
8680 return std::nullopt;
8684 NumBytesToCreateFrame += 8;
8691 return std::nullopt;
8694 NumBytesToCreateFrame, FrameID);
8698 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
8702 const auto &CFn = Candidates.front().getMF()->getFunction();
8706 if (CFn.hasFnAttribute(
"sign-return-address"))
8707 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
8708 if (CFn.hasFnAttribute(
"sign-return-address-key"))
8709 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
8711 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
8719 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
8733 if (!AFI || AFI->
hasRedZone().value_or(
true))
8753 unsigned &Flags)
const {
8755 "Must track liveness!");
8757 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
8772 auto AreAllUnsafeRegsDead = [&LRU]() {
8789 bool LRAvailableEverywhere =
true;
8794 if (
MI.isCall() && !
MI.isTerminator())
8795 Flags |= MachineOutlinerMBBFlags::HasCalls;
8800 auto CreateNewRangeStartingAt =
8801 [&RangeBegin, &RangeEnd,
8803 RangeBegin = NewBegin;
8804 RangeEnd = std::next(RangeBegin);
8807 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
8812 Ranges.push_back(std::make_pair(RangeBegin, RangeEnd));
8820 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
8824 UpdateWholeMBBFlags(*FirstPossibleEndPt);
8825 if (AreAllUnsafeRegsDead())
8832 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
8838 UpdateWholeMBBFlags(
MI);
8839 if (!AreAllUnsafeRegsDead()) {
8840 SaveRangeIfNonEmpty();
8841 CreateNewRangeStartingAt(
MI.getIterator());
8844 LRAvailableEverywhere &= LRU.
available(AArch64::LR);
8845 RangeBegin =
MI.getIterator();
8850 if (AreAllUnsafeRegsDead())
8851 SaveRangeIfNonEmpty();
8856 std::reverse(Ranges.begin(), Ranges.end());
8859 if (!LRAvailableEverywhere)
8860 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
8867 unsigned Flags)
const {
8875 switch (
MI.getOpcode()) {
8877 case AArch64::PACIASP:
8878 case AArch64::PACIBSP:
8879 case AArch64::PACIASPPC:
8880 case AArch64::PACIBSPPC:
8881 case AArch64::AUTIASP:
8882 case AArch64::AUTIBSP:
8883 case AArch64::AUTIASPPCi:
8884 case AArch64::AUTIASPPCr:
8885 case AArch64::AUTIBSPPCi:
8886 case AArch64::AUTIBSPPCr:
8887 case AArch64::RETAA:
8888 case AArch64::RETAB:
8889 case AArch64::RETAASPPCi:
8890 case AArch64::RETAASPPCr:
8891 case AArch64::RETABSPPCi:
8892 case AArch64::RETABSPPCr:
8893 case AArch64::EMITBKEY:
8894 case AArch64::PAUTH_PROLOGUE:
8895 case AArch64::PAUTH_EPILOGUE:
8909 if (
MI.isCFIInstruction())
8913 if (
MI.isTerminator())
8922 assert(!MOP.isCFIIndex());
8925 if (MOP.isReg() && !MOP.isImplicit() &&
8926 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
8933 if (
MI.getOpcode() == AArch64::ADRP)
8954 if (MOP.isGlobal()) {
8955 Callee = dyn_cast<Function>(MOP.getGlobal());
8962 if (Callee && Callee->getName() ==
"\01_mcount")
8970 if (
MI.getOpcode() == AArch64::BLR ||
8971 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
8975 return UnknownCallOutlineType;
8983 return UnknownCallOutlineType;
8991 return UnknownCallOutlineType;
9016 bool OffsetIsScalable;
9019 if (!
MI.mayLoadOrStore() ||
9022 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
9027 int64_t Dummy1, Dummy2;
9030 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
9032 assert(Scale != 0 &&
"Unexpected opcode!");
9033 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
9038 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
9039 StackOffsetOperand.
setImm(NewImm);
9045 bool ShouldSignReturnAddr) {
9046 if (!ShouldSignReturnAddr)
9052 TII->get(AArch64::PAUTH_EPILOGUE))
9068 unsigned TailOpcode;
9069 if (Call->getOpcode() == AArch64::BL) {
9070 TailOpcode = AArch64::TCRETURNdi;
9072 assert(Call->getOpcode() == AArch64::BLR ||
9073 Call->getOpcode() == AArch64::BLRNoIP);
9074 TailOpcode = AArch64::TCRETURNriALL;
9077 .
add(Call->getOperand(0))
9080 Call->eraseFromParent();
9085 bool IsLeafFunction =
true;
9089 return MI.isCall() && !
MI.isReturn();
9099 "Can only fix up stack references once");
9100 fixupPostOutline(
MBB);
9102 IsLeafFunction =
false;
9113 Et = std::prev(
MBB.
end());
9126 unsigned DwarfReg =
MRI->getDwarfRegNum(AArch64::LR,
true);
9129 int64_t StackPosEntry =
9182 fixupPostOutline(
MBB);
9193 .addGlobalAddress(M.getNamedValue(MF.
getName()))
9203 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9218 Register Reg = findRegisterToSaveLRTo(
C);
9219 assert(Reg &&
"No callee-saved register available?");
9253 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9269 bool AllowSideEffects)
const {
9274 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
9276 }
else if (STI.hasSVE()) {
9286std::optional<DestSourcePair>
9291 if (
MI.getOpcode() == AArch64::ORRWrs &&
9292 MI.getOperand(1).getReg() == AArch64::WZR &&
9293 MI.getOperand(3).getImm() == 0x0 &&
9295 (!
MI.getOperand(0).getReg().isVirtual() ||
9296 MI.getOperand(0).getSubReg() == 0) &&
9297 (!
MI.getOperand(0).getReg().isPhysical() ||
9298 MI.findRegisterDefOperandIdx(
MI.getOperand(0).getReg() - AArch64::W0 +
9303 if (
MI.getOpcode() == AArch64::ORRXrs &&
9304 MI.getOperand(1).getReg() == AArch64::XZR &&
9305 MI.getOperand(3).getImm() == 0x0)
9308 return std::nullopt;
9311std::optional<DestSourcePair>
9313 if (
MI.getOpcode() == AArch64::ORRWrs &&
9314 MI.getOperand(1).getReg() == AArch64::WZR &&
9315 MI.getOperand(3).getImm() == 0x0)
9317 return std::nullopt;
9320std::optional<RegImmPair>
9329 return std::nullopt;
9331 switch (
MI.getOpcode()) {
9333 return std::nullopt;
9334 case AArch64::SUBWri:
9335 case AArch64::SUBXri:
9336 case AArch64::SUBSWri:
9337 case AArch64::SUBSXri:
9340 case AArch64::ADDSWri:
9341 case AArch64::ADDSXri:
9342 case AArch64::ADDWri:
9343 case AArch64::ADDXri: {
9345 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
9346 !
MI.getOperand(2).isImm())
9347 return std::nullopt;
9348 int Shift =
MI.getOperand(3).getImm();
9349 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
9350 Offset = Sign * (
MI.getOperand(2).getImm() << Shift);
9359static std::optional<ParamLoadedValue>
9363 auto DestSrc =
TII->isCopyLikeInstr(
MI);
9365 return std::nullopt;
9367 Register DestReg = DestSrc->Destination->getReg();
9368 Register SrcReg = DestSrc->Source->getReg();
9373 if (DestReg == DescribedReg)
9377 if (
MI.getOpcode() == AArch64::ORRWrs &&
9378 TRI->isSuperRegister(DestReg, DescribedReg))
9382 if (
MI.getOpcode() == AArch64::ORRXrs &&
9383 TRI->isSubRegister(DestReg, DescribedReg)) {
9384 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
9388 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
9389 "Unhandled ORR[XW]rs copy case");
9391 return std::nullopt;
9410 return MI.getOpcode() == AArch64::INLINEASM_BR;
9428 switch (
MI.getOpcode()) {
9429 case TargetOpcode::G_BRJT:
9430 case AArch64::JumpTableDest32:
9431 case AArch64::JumpTableDest16:
9432 case AArch64::JumpTableDest8:
9443std::optional<ParamLoadedValue>
9448 switch (
MI.getOpcode()) {
9449 case AArch64::MOVZWi:
9450 case AArch64::MOVZXi: {
9453 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
9454 return std::nullopt;
9456 if (!
MI.getOperand(1).isImm())
9457 return std::nullopt;
9458 int64_t Immediate =
MI.getOperand(1).getImm();
9459 int Shift =
MI.getOperand(2).getImm();
9463 case AArch64::ORRWrs:
9464 case AArch64::ORRXrs:
9474 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
9475 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
9478 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
9482 if (!
MRI.hasOneNonDBGUse(DefReg))
9487 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
9488 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
9509 unsigned Scale)
const {
9520 unsigned Shift =
Log2_64(NumBytes);
9521 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
9529 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
9534 return AArch64::BLRNoIP;
9536 return AArch64::BLR;
9541 Register TargetReg,
bool FrameSetup)
const {
9542 assert(TargetReg != AArch64::SP &&
"New top of stack cannot aleady be in SP");
9554 MF.
insert(MBBInsertPoint, LoopTestMBB);
9557 MF.
insert(MBBInsertPoint, LoopBodyMBB);
9559 MF.
insert(MBBInsertPoint, ExitMBB);
9569 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
9583 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
9596 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
9621 return ExitMBB->
begin();
9638 unsigned CompCounterOprNum;
9642 unsigned UpdateCounterOprNum;
9646 bool IsUpdatePriorComp;
9658 TII(MF->getSubtarget().getInstrInfo()),
9659 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
9660 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
9661 CompCounterOprNum(CompCounterOprNum), Update(Update),
9662 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
9663 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
9665 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
9671 std::optional<bool> createTripCountGreaterCondition(
9681 void createRemainingIterationsGreaterCondition(
9687 void adjustTripCount(
int TripCountAdjust)
override {}
9689 void disposed()
override {}
9690 bool isMVEExpanderSupported()
override {
return true; }
9708 Result =
MRI.createVirtualRegister(
9711 }
else if (
I == ReplaceOprNum) {
9712 MRI.constrainRegClass(
9722void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
9738 assert(CondBranch->getOpcode() == AArch64::Bcc);
9742 if (CondBranch->getOperand(1).getMBB() == LoopBB)
9749 auto AccumulateCond = [&](
Register CurCond,
9751 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
9760 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
9764 for (
int I = 0;
I <= TC; ++
I) {
9770 AccCond = AccumulateCond(AccCond,
CC);
9774 if (Update != Comp && IsUpdatePriorComp) {
9776 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
9777 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
9781 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
9783 }
else if (Update != Comp) {
9788 Counter = NextCounter;
9792 if (LastStage0Insts.
empty()) {
9796 if (IsUpdatePriorComp)
9801 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
9804 for (
int I = 0;
I <= TC; ++
I) {
9808 AccCond = AccumulateCond(AccCond,
CC);
9809 if (
I != TC && Update != Comp)
9812 Counter = NextCounter;
9828 assert(Phi.getNumOperands() == 5);
9829 if (Phi.getOperand(2).getMBB() ==
MBB) {
9830 RegMBB = Phi.getOperand(1).getReg();
9831 RegOther = Phi.getOperand(3).getReg();
9833 assert(Phi.getOperand(4).getMBB() ==
MBB);
9834 RegMBB = Phi.getOperand(3).getReg();
9835 RegOther = Phi.getOperand(1).getReg();
9840 if (!Reg.isVirtual())
9843 return MRI.getVRegDef(Reg)->getParent() != BB;
9849 unsigned &UpdateCounterOprNum,
Register &InitReg,
9850 bool &IsUpdatePriorComp) {
9864 if (!Reg.isVirtual())
9867 UpdateInst =
nullptr;
9868 UpdateCounterOprNum = 0;
9870 IsUpdatePriorComp =
true;
9874 if (Def->getParent() != LoopBB)
9876 if (Def->isCopy()) {
9878 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
9880 CurReg = Def->getOperand(1).getReg();
9881 }
else if (Def->isPHI()) {
9885 IsUpdatePriorComp =
false;
9890 switch (Def->getOpcode()) {
9891 case AArch64::ADDSXri:
9892 case AArch64::ADDSWri:
9893 case AArch64::SUBSXri:
9894 case AArch64::SUBSWri:
9895 case AArch64::ADDXri:
9896 case AArch64::ADDWri:
9897 case AArch64::SUBXri:
9898 case AArch64::SUBWri:
9900 UpdateCounterOprNum = 1;
9902 case AArch64::ADDSXrr:
9903 case AArch64::ADDSWrr:
9904 case AArch64::SUBSXrr:
9905 case AArch64::SUBSWrr:
9906 case AArch64::ADDXrr:
9907 case AArch64::ADDWrr:
9908 case AArch64::SUBXrr:
9909 case AArch64::SUBWrr:
9912 UpdateCounterOprNum = 1;
9914 UpdateCounterOprNum = 2;
9921 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
9936std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
9947 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
9958 if (
TBB == LoopBB && FBB == LoopBB)
9962 if (
TBB != LoopBB && FBB ==
nullptr)
9965 assert((
TBB == LoopBB || FBB == LoopBB) &&
9966 "The Loop must be a single-basic-block loop");
9971 if (CondBranch->
getOpcode() != AArch64::Bcc)
9979 unsigned CompCounterOprNum = 0;
9981 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
9985 switch (
MI.getOpcode()) {
9986 case AArch64::SUBSXri:
9987 case AArch64::SUBSWri:
9988 case AArch64::ADDSXri:
9989 case AArch64::ADDSWri:
9991 CompCounterOprNum = 1;
9993 case AArch64::ADDSWrr:
9994 case AArch64::ADDSXrr:
9995 case AArch64::SUBSWrr:
9996 case AArch64::SUBSXrr:
10007 if (CompCounterOprNum == 0) {
10009 CompCounterOprNum = 2;
10011 CompCounterOprNum = 1;
10023 bool IsUpdatePriorComp;
10024 unsigned UpdateCounterOprNum;
10026 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
10029 return std::make_unique<AArch64PipelinerLoopInfo>(
10030 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
10034#define GET_INSTRINFO_HELPERS
10035#define GET_INSTRMAP_INFO
10036#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static unsigned getBranchDisplacementBits(unsigned Opc)
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, unsigned Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
const SetOfInstructions & getLOHRelated() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
void setOutliningStyle(std::string Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors, int64_t &NumDataVectors)
Returns the offset in parts to which this frame offset can be decomposed for the purpose of describin...
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
uint64_t getElementSizeForOpcode(unsigned Opc) const
Returns the vector element size (B, H, S or D) of an SVE opcode.
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
bool isWhileOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE WHILE## instruction.
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
SmallVector< std::pair< MachineBasicBlock::iterator, MachineBasicBlock::iterator > > getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool useMachineCombiner() const override
AArch64 supports MachineCombiner.
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
std::optional< outliner::OutlinedFunction > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs) const override
bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const override
static bool isFalkorShiftExtFast(const MachineInstr &MI)
Returns true if the instruction has a shift by immediate that can be executed in one cycle less.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
bool expandPostRAPseudo(MachineInstr &MI) const override
unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL, unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
bool isFunctionSafeToSplit(const MachineFunction &MF) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
Return true when Inst is associative and commutative so that it can be reassociated.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset, unsigned Scale) const
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
ArrayRef< std::pair< unsigned, const char * > > getSerializableBitmaskMachineOperandTargetFlags() const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isPTestLikeOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE instruction that sets the condition codes as if it's results...
void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const override
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isTargetILP32() const
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MBBSectionID getSectionID() const
Returns the section ID of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
Primary interface to the complete machine description for the target machine.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool succeeded(LogicalResult Result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
static bool isIndirectBranchOpcode(int Opc)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
unsigned getUndefRegState(bool B)
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
DWARFExpression::Operation Op
static bool isUncondBranchOpcode(int Opc)
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
static const MachineMemOperand::Flags MOStridedAccess
@ Default
The result values are uniform if and only if all operands are uniform.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
static const MBBSectionID ColdSectionID
MachineJumpTableEntry - One jump table in the jump table info.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
MachineBasicBlock::iterator begin()
MachineBasicBlock::iterator end()
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.