63#define GET_INSTRINFO_CTOR_DTOR
64#include "AArch64GenInstrInfo.inc"
68 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
72 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
76 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
80 cl::desc(
"Restrict range of B instructions (DEBUG)"));
85 RI(STI.getTargetTriple()), Subtarget(STI) {}
96 auto Op =
MI.getOpcode();
97 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
98 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
102 if (
MI.isMetaInstruction())
107 unsigned NumBytes = 0;
113 switch (
Desc.getOpcode()) {
116 return Desc.getSize();
123 case TargetOpcode::STACKMAP:
126 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
128 case TargetOpcode::PATCHPOINT:
131 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
133 case TargetOpcode::STATEPOINT:
135 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
140 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
145 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
147 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
148 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
152 case TargetOpcode::PATCHABLE_EVENT_CALL:
158 NumBytes =
MI.getOperand(1).getImm();
160 case TargetOpcode::BUNDLE:
161 NumBytes = getInstBundleLength(
MI);
168unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
172 while (++
I != E &&
I->isInsideBundle()) {
173 assert(!
I->isBundle() &&
"No nested bundle!");
232 int64_t BrOffset)
const {
234 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
235 "over conditional branch expansion");
236 return isIntN(Bits, BrOffset / 4);
241 switch (
MI.getOpcode()) {
245 return MI.getOperand(0).getMBB();
250 return MI.getOperand(2).getMBB();
256 return MI.getOperand(1).getMBB();
266 assert(RS &&
"RegScavenger required for long branching");
268 "new block should be inserted for expanding unconditional branch");
271 "restore block should be inserted for restoring clobbered registers");
276 if (!isInt<33>(BrOffset))
278 "Branch offsets outside of the signed 33-bit range not supported");
292 constexpr Register Reg = AArch64::X16;
294 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
302 if (Scavenged != AArch64::NoRegister &&
304 buildIndirectBranch(Scavenged, NewDestBB);
314 "Unable to insert indirect branch inside function that has red zone");
337 bool AllowModify)
const {
344 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
345 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
349 if (!isUnpredicatedTerminator(*
I))
356 unsigned LastOpc = LastInst->
getOpcode();
357 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
372 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
379 LastInst = SecondLastInst;
381 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
386 SecondLastInst = &*
I;
387 SecondLastOpc = SecondLastInst->
getOpcode();
398 LastInst = SecondLastInst;
400 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
402 "unreachable unconditional branches removed above");
411 SecondLastInst = &*
I;
412 SecondLastOpc = SecondLastInst->
getOpcode();
416 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
432 I->eraseFromParent();
441 I->eraseFromParent();
450 MachineBranchPredicate &MBP,
451 bool AllowModify)
const {
461 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
462 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
466 if (!isUnpredicatedTerminator(*
I))
471 unsigned LastOpc = LastInst->
getOpcode();
486 assert(MBP.TrueDest &&
"expected!");
489 MBP.ConditionDef =
nullptr;
490 MBP.SingleUseCondition =
false;
494 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
495 : MachineBranchPredicate::PRED_EQ;
501 if (
Cond[0].getImm() != -1) {
507 switch (
Cond[1].getImm()) {
511 Cond[1].setImm(AArch64::CBNZW);
514 Cond[1].setImm(AArch64::CBZW);
517 Cond[1].setImm(AArch64::CBNZX);
520 Cond[1].setImm(AArch64::CBZX);
523 Cond[1].setImm(AArch64::TBNZW);
526 Cond[1].setImm(AArch64::TBZW);
529 Cond[1].setImm(AArch64::TBNZX);
532 Cond[1].setImm(AArch64::TBZX);
541 int *BytesRemoved)
const {
551 I->eraseFromParent();
568 I->eraseFromParent();
575void AArch64InstrInfo::instantiateCondBranch(
578 if (
Cond[0].getImm() != -1) {
596 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
635 unsigned *NewVReg =
nullptr) {
640 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
643 unsigned SrcOpNum = 0;
645 case AArch64::ADDSXri:
646 case AArch64::ADDSWri:
653 case AArch64::ADDXri:
654 case AArch64::ADDWri:
660 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
663 case AArch64::ORNXrr:
664 case AArch64::ORNWrr: {
667 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
670 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
674 case AArch64::SUBSXrr:
675 case AArch64::SUBSWrr:
682 case AArch64::SUBXrr:
683 case AArch64::SUBWrr: {
686 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
689 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
695 assert(Opc && SrcOpNum &&
"Missing parameters");
707 int &FalseCycles)
const {
711 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
718 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
722 unsigned ExtraCondLat =
Cond.size() != 1;
726 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
727 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
729 CondCycles = 1 + ExtraCondLat;
730 TrueCycles = FalseCycles = 1;
740 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
741 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
742 CondCycles = 5 + ExtraCondLat;
743 TrueCycles = FalseCycles = 2;
760 switch (
Cond.size()) {
769 switch (
Cond[1].getImm()) {
792 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
798 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
808 switch (
Cond[1].getImm()) {
821 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
837 bool TryFold =
false;
838 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
839 RC = &AArch64::GPR64RegClass;
840 Opc = AArch64::CSELXr;
842 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
843 RC = &AArch64::GPR32RegClass;
844 Opc = AArch64::CSELWr;
846 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
847 RC = &AArch64::FPR64RegClass;
848 Opc = AArch64::FCSELDrrr;
849 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
850 RC = &AArch64::FPR32RegClass;
851 Opc = AArch64::FCSELSrrr;
853 assert(RC &&
"Unsupported regclass");
857 unsigned NewVReg = 0;
872 MRI.clearKillFlags(NewVReg);
877 MRI.constrainRegClass(TrueReg, RC);
878 MRI.constrainRegClass(FalseReg, RC);
893 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
898 return Is.
size() <= 2;
904 if (Subtarget.hasExynosCheapAsMoveHandling()) {
905 if (isExynosCheapAsMove(
MI))
907 return MI.isAsCheapAsAMove();
910 switch (
MI.getOpcode()) {
912 return MI.isAsCheapAsAMove();
914 case AArch64::ADDWrs:
915 case AArch64::ADDXrs:
916 case AArch64::SUBWrs:
917 case AArch64::SUBXrs:
918 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
923 case AArch64::MOVi32imm:
925 case AArch64::MOVi64imm:
931 switch (
MI.getOpcode()) {
935 case AArch64::ADDWrs:
936 case AArch64::ADDXrs:
937 case AArch64::ADDSWrs:
938 case AArch64::ADDSXrs: {
939 unsigned Imm =
MI.getOperand(3).getImm();
946 case AArch64::ADDWrx:
947 case AArch64::ADDXrx:
948 case AArch64::ADDXrx64:
949 case AArch64::ADDSWrx:
950 case AArch64::ADDSXrx:
951 case AArch64::ADDSXrx64: {
952 unsigned Imm =
MI.getOperand(3).getImm();
964 case AArch64::SUBWrs:
965 case AArch64::SUBSWrs: {
966 unsigned Imm =
MI.getOperand(3).getImm();
968 return ShiftVal == 0 ||
972 case AArch64::SUBXrs:
973 case AArch64::SUBSXrs: {
974 unsigned Imm =
MI.getOperand(3).getImm();
976 return ShiftVal == 0 ||
980 case AArch64::SUBWrx:
981 case AArch64::SUBXrx:
982 case AArch64::SUBXrx64:
983 case AArch64::SUBSWrx:
984 case AArch64::SUBSXrx:
985 case AArch64::SUBSXrx64: {
986 unsigned Imm =
MI.getOperand(3).getImm();
998 case AArch64::LDRBBroW:
999 case AArch64::LDRBBroX:
1000 case AArch64::LDRBroW:
1001 case AArch64::LDRBroX:
1002 case AArch64::LDRDroW:
1003 case AArch64::LDRDroX:
1004 case AArch64::LDRHHroW:
1005 case AArch64::LDRHHroX:
1006 case AArch64::LDRHroW:
1007 case AArch64::LDRHroX:
1008 case AArch64::LDRQroW:
1009 case AArch64::LDRQroX:
1010 case AArch64::LDRSBWroW:
1011 case AArch64::LDRSBWroX:
1012 case AArch64::LDRSBXroW:
1013 case AArch64::LDRSBXroX:
1014 case AArch64::LDRSHWroW:
1015 case AArch64::LDRSHWroX:
1016 case AArch64::LDRSHXroW:
1017 case AArch64::LDRSHXroX:
1018 case AArch64::LDRSWroW:
1019 case AArch64::LDRSWroX:
1020 case AArch64::LDRSroW:
1021 case AArch64::LDRSroX:
1022 case AArch64::LDRWroW:
1023 case AArch64::LDRWroX:
1024 case AArch64::LDRXroW:
1025 case AArch64::LDRXroX:
1026 case AArch64::PRFMroW:
1027 case AArch64::PRFMroX:
1028 case AArch64::STRBBroW:
1029 case AArch64::STRBBroX:
1030 case AArch64::STRBroW:
1031 case AArch64::STRBroX:
1032 case AArch64::STRDroW:
1033 case AArch64::STRDroX:
1034 case AArch64::STRHHroW:
1035 case AArch64::STRHHroX:
1036 case AArch64::STRHroW:
1037 case AArch64::STRHroX:
1038 case AArch64::STRQroW:
1039 case AArch64::STRQroX:
1040 case AArch64::STRSroW:
1041 case AArch64::STRSroX:
1042 case AArch64::STRWroW:
1043 case AArch64::STRWroX:
1044 case AArch64::STRXroW:
1045 case AArch64::STRXroX: {
1046 unsigned IsSigned =
MI.getOperand(3).getImm();
1053 unsigned Opc =
MI.getOpcode();
1057 case AArch64::SEH_StackAlloc:
1058 case AArch64::SEH_SaveFPLR:
1059 case AArch64::SEH_SaveFPLR_X:
1060 case AArch64::SEH_SaveReg:
1061 case AArch64::SEH_SaveReg_X:
1062 case AArch64::SEH_SaveRegP:
1063 case AArch64::SEH_SaveRegP_X:
1064 case AArch64::SEH_SaveFReg:
1065 case AArch64::SEH_SaveFReg_X:
1066 case AArch64::SEH_SaveFRegP:
1067 case AArch64::SEH_SaveFRegP_X:
1068 case AArch64::SEH_SetFP:
1069 case AArch64::SEH_AddFP:
1070 case AArch64::SEH_Nop:
1071 case AArch64::SEH_PrologEnd:
1072 case AArch64::SEH_EpilogStart:
1073 case AArch64::SEH_EpilogEnd:
1074 case AArch64::SEH_PACSignLR:
1075 case AArch64::SEH_SaveAnyRegQP:
1076 case AArch64::SEH_SaveAnyRegQPX:
1083 unsigned &SubIdx)
const {
1084 switch (
MI.getOpcode()) {
1087 case AArch64::SBFMXri:
1088 case AArch64::UBFMXri:
1091 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1094 SrcReg =
MI.getOperand(1).getReg();
1095 DstReg =
MI.getOperand(0).getReg();
1096 SubIdx = AArch64::sub_32;
1105 int64_t OffsetA = 0, OffsetB = 0;
1106 TypeSize WidthA(0,
false), WidthB(0,
false);
1107 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1128 OffsetAIsScalable == OffsetBIsScalable) {
1129 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1130 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1131 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1132 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1150 switch (
MI.getOpcode()) {
1153 if (
MI.getOperand(0).getImm() == 0x14)
1160 case AArch64::MSRpstatesvcrImm1:
1167 auto Next = std::next(
MI.getIterator());
1168 return Next !=
MBB->
end() && Next->isCFIInstruction();
1175 Register &SrcReg2, int64_t &CmpMask,
1176 int64_t &CmpValue)
const {
1179 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1180 if (!
MI.getOperand(1).isReg())
1183 switch (
MI.getOpcode()) {
1186 case AArch64::PTEST_PP:
1187 case AArch64::PTEST_PP_ANY:
1188 SrcReg =
MI.getOperand(0).getReg();
1189 SrcReg2 =
MI.getOperand(1).getReg();
1194 case AArch64::SUBSWrr:
1195 case AArch64::SUBSWrs:
1196 case AArch64::SUBSWrx:
1197 case AArch64::SUBSXrr:
1198 case AArch64::SUBSXrs:
1199 case AArch64::SUBSXrx:
1200 case AArch64::ADDSWrr:
1201 case AArch64::ADDSWrs:
1202 case AArch64::ADDSWrx:
1203 case AArch64::ADDSXrr:
1204 case AArch64::ADDSXrs:
1205 case AArch64::ADDSXrx:
1207 SrcReg =
MI.getOperand(1).getReg();
1208 SrcReg2 =
MI.getOperand(2).getReg();
1212 case AArch64::SUBSWri:
1213 case AArch64::ADDSWri:
1214 case AArch64::SUBSXri:
1215 case AArch64::ADDSXri:
1216 SrcReg =
MI.getOperand(1).getReg();
1219 CmpValue =
MI.getOperand(2).getImm();
1221 case AArch64::ANDSWri:
1222 case AArch64::ANDSXri:
1225 SrcReg =
MI.getOperand(1).getReg();
1229 MI.getOperand(2).getImm(),
1230 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1239 assert(
MBB &&
"Can't get MachineBasicBlock here");
1241 assert(MF &&
"Can't get MachineFunction here");
1246 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1250 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1253 if (!OpRegCstraints)
1261 "Operand has register constraints without being a register!");
1264 if (Reg.isPhysical()) {
1265 if (!OpRegCstraints->
contains(Reg))
1268 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1281 bool MIDefinesZeroReg =
false;
1282 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1283 MI.definesRegister(AArch64::XZR,
nullptr))
1284 MIDefinesZeroReg =
true;
1286 switch (
MI.getOpcode()) {
1288 return MI.getOpcode();
1289 case AArch64::ADDSWrr:
1290 return AArch64::ADDWrr;
1291 case AArch64::ADDSWri:
1292 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1293 case AArch64::ADDSWrs:
1294 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1295 case AArch64::ADDSWrx:
1296 return AArch64::ADDWrx;
1297 case AArch64::ADDSXrr:
1298 return AArch64::ADDXrr;
1299 case AArch64::ADDSXri:
1300 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1301 case AArch64::ADDSXrs:
1302 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1303 case AArch64::ADDSXrx:
1304 return AArch64::ADDXrx;
1305 case AArch64::SUBSWrr:
1306 return AArch64::SUBWrr;
1307 case AArch64::SUBSWri:
1308 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1309 case AArch64::SUBSWrs:
1310 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1311 case AArch64::SUBSWrx:
1312 return AArch64::SUBWrx;
1313 case AArch64::SUBSXrr:
1314 return AArch64::SUBXrr;
1315 case AArch64::SUBSXri:
1316 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1317 case AArch64::SUBSXrs:
1318 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1319 case AArch64::SUBSXrx:
1320 return AArch64::SUBXrx;
1335 if (To == To->getParent()->begin())
1340 if (To->getParent() !=
From->getParent())
1352 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1353 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1359std::optional<unsigned>
1363 unsigned MaskOpcode =
Mask->getOpcode();
1364 unsigned PredOpcode = Pred->
getOpcode();
1368 if (PredIsWhileLike) {
1372 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1386 if (PredIsPTestLike) {
1391 if ((Mask == Pred) && PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1401 if (Mask == PTestLikeMask || PTest->
getOpcode() == AArch64::PTEST_PP_ANY)
1430 PTest->
getOpcode() == AArch64::PTEST_PP_ANY))
1438 switch (PredOpcode) {
1439 case AArch64::AND_PPzPP:
1440 case AArch64::BIC_PPzPP:
1441 case AArch64::EOR_PPzPP:
1442 case AArch64::NAND_PPzPP:
1443 case AArch64::NOR_PPzPP:
1444 case AArch64::ORN_PPzPP:
1445 case AArch64::ORR_PPzPP:
1446 case AArch64::BRKA_PPzP:
1447 case AArch64::BRKPA_PPzPP:
1448 case AArch64::BRKB_PPzP:
1449 case AArch64::BRKPB_PPzPP:
1450 case AArch64::RDFFR_PPz: {
1454 if (Mask != PredMask)
1458 case AArch64::BRKN_PPzP: {
1462 if ((MaskOpcode != AArch64::PTRUE_B) ||
1463 (
Mask->getOperand(1).getImm() != 31))
1467 case AArch64::PTRUE_B:
1480bool AArch64InstrInfo::optimizePTestInstr(
1481 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1483 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1484 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1485 unsigned PredOpcode = Pred->
getOpcode();
1486 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred,
MRI);
1502 if (*NewOp != PredOpcode) {
1506 assert(succeeded &&
"Operands have incompatible register classes!");
1513 for (; i !=
e; ++i) {
1544 if (DeadNZCVIdx != -1) {
1559 assert(succeeded &&
"Some operands reg class are incompatible!");
1563 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1564 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1565 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1574 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1576 return (CmpValue == 0 || CmpValue == 1) &&
1577 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1585 switch (Instr.getOpcode()) {
1587 return AArch64::INSTRUCTION_LIST_END;
1589 case AArch64::ADDSWrr:
1590 case AArch64::ADDSWri:
1591 case AArch64::ADDSXrr:
1592 case AArch64::ADDSXri:
1593 case AArch64::SUBSWrr:
1594 case AArch64::SUBSWri:
1595 case AArch64::SUBSXrr:
1596 case AArch64::SUBSXri:
1597 return Instr.getOpcode();
1599 case AArch64::ADDWrr:
1600 return AArch64::ADDSWrr;
1601 case AArch64::ADDWri:
1602 return AArch64::ADDSWri;
1603 case AArch64::ADDXrr:
1604 return AArch64::ADDSXrr;
1605 case AArch64::ADDXri:
1606 return AArch64::ADDSXri;
1607 case AArch64::ADCWr:
1608 return AArch64::ADCSWr;
1609 case AArch64::ADCXr:
1610 return AArch64::ADCSXr;
1611 case AArch64::SUBWrr:
1612 return AArch64::SUBSWrr;
1613 case AArch64::SUBWri:
1614 return AArch64::SUBSWri;
1615 case AArch64::SUBXrr:
1616 return AArch64::SUBSXrr;
1617 case AArch64::SUBXri:
1618 return AArch64::SUBSXri;
1619 case AArch64::SBCWr:
1620 return AArch64::SBCSWr;
1621 case AArch64::SBCXr:
1622 return AArch64::SBCSXr;
1623 case AArch64::ANDWri:
1624 return AArch64::ANDSWri;
1625 case AArch64::ANDXri:
1626 return AArch64::ANDSXri;
1633 if (BB->isLiveIn(AArch64::NZCV))
1642 switch (Instr.getOpcode()) {
1646 case AArch64::Bcc: {
1647 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1652 case AArch64::CSINVWr:
1653 case AArch64::CSINVXr:
1654 case AArch64::CSINCWr:
1655 case AArch64::CSINCXr:
1656 case AArch64::CSELWr:
1657 case AArch64::CSELXr:
1658 case AArch64::CSNEGWr:
1659 case AArch64::CSNEGXr:
1660 case AArch64::FCSELSrrr:
1661 case AArch64::FCSELDrrr: {
1662 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1675 Instr.getOperand(CCIdx).getImm())
1728std::optional<UsedNZCV>
1733 if (
MI.getParent() != CmpParent)
1734 return std::nullopt;
1737 return std::nullopt;
1742 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1745 return std::nullopt;
1750 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1753 return NZCVUsedAfterCmp;
1757 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1761 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1783 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1789 "Caller guarantees that CmpInstr compares with constant 0");
1792 if (!NZVCUsed || NZVCUsed->C)
1814bool AArch64InstrInfo::substituteCmpToZero(
1825 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1832 MI->setDesc(
get(NewOpc));
1836 assert(succeeded &&
"Some operands reg class are incompatible!");
1837 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1849 assert((CmpValue == 0 || CmpValue == 1) &&
1850 "Only comparisons to 0 or 1 considered for removal!");
1853 unsigned MIOpc =
MI.getOpcode();
1854 if (MIOpc == AArch64::CSINCWr) {
1855 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1856 MI.getOperand(2).getReg() != AArch64::WZR)
1858 }
else if (MIOpc == AArch64::CSINCXr) {
1859 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1860 MI.getOperand(2).getReg() != AArch64::XZR)
1870 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1874 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1876 if (CmpValue && !IsSubsRegImm)
1878 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1883 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1886 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1890 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1893 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1894 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1897 if (MIUsedNZCV.
N && !CmpValue)
1939bool AArch64InstrInfo::removeCmpToZeroOrOne(
1947 bool IsInvertCC =
false;
1957 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1968 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1969 MI.getOpcode() != AArch64::CATCHRET)
1977 if (
MI.getOpcode() == AArch64::CATCHRET) {
1986 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1988 FirstEpilogSEH = std::next(FirstEpilogSEH);
2002 if (M.getStackProtectorGuard() ==
"sysreg") {
2012 int Offset = M.getStackProtectorGuardOffset();
2064 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2073 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2115 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2138 switch (
MI.getOpcode()) {
2141 case AArch64::MOVZWi:
2142 case AArch64::MOVZXi:
2143 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2144 assert(
MI.getDesc().getNumOperands() == 3 &&
2145 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2149 case AArch64::ANDWri:
2150 return MI.getOperand(1).getReg() == AArch64::WZR;
2151 case AArch64::ANDXri:
2152 return MI.getOperand(1).getReg() == AArch64::XZR;
2153 case TargetOpcode::COPY:
2154 return MI.getOperand(1).getReg() == AArch64::WZR;
2162 switch (
MI.getOpcode()) {
2165 case TargetOpcode::COPY: {
2168 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2169 AArch64::GPR64RegClass.
contains(DstReg));
2171 case AArch64::ORRXrs:
2172 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2173 assert(
MI.getDesc().getNumOperands() == 4 &&
2174 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2178 case AArch64::ADDXri:
2179 if (
MI.getOperand(2).getImm() == 0) {
2180 assert(
MI.getDesc().getNumOperands() == 4 &&
2181 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2192 switch (
MI.getOpcode()) {
2195 case TargetOpcode::COPY: {
2197 return AArch64::FPR128RegClass.contains(DstReg);
2199 case AArch64::ORRv16i8:
2200 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2201 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2202 "invalid ORRv16i8 operands");
2211 int &FrameIndex)
const {
2212 switch (
MI.getOpcode()) {
2215 case AArch64::LDRWui:
2216 case AArch64::LDRXui:
2217 case AArch64::LDRBui:
2218 case AArch64::LDRHui:
2219 case AArch64::LDRSui:
2220 case AArch64::LDRDui:
2221 case AArch64::LDRQui:
2222 case AArch64::LDR_PXI:
2223 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2224 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2225 FrameIndex =
MI.getOperand(1).getIndex();
2226 return MI.getOperand(0).getReg();
2235 int &FrameIndex)
const {
2236 switch (
MI.getOpcode()) {
2239 case AArch64::STRWui:
2240 case AArch64::STRXui:
2241 case AArch64::STRBui:
2242 case AArch64::STRHui:
2243 case AArch64::STRSui:
2244 case AArch64::STRDui:
2245 case AArch64::STRQui:
2246 case AArch64::STR_PXI:
2247 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2248 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2249 FrameIndex =
MI.getOperand(1).getIndex();
2250 return MI.getOperand(0).getReg();
2260 return MMO->getFlags() & MOSuppressPair;
2266 if (
MI.memoperands_empty())
2274 return MMO->getFlags() & MOStridedAccess;
2282 case AArch64::STURSi:
2283 case AArch64::STRSpre:
2284 case AArch64::STURDi:
2285 case AArch64::STRDpre:
2286 case AArch64::STURQi:
2287 case AArch64::STRQpre:
2288 case AArch64::STURBBi:
2289 case AArch64::STURHHi:
2290 case AArch64::STURWi:
2291 case AArch64::STRWpre:
2292 case AArch64::STURXi:
2293 case AArch64::STRXpre:
2294 case AArch64::LDURSi:
2295 case AArch64::LDRSpre:
2296 case AArch64::LDURDi:
2297 case AArch64::LDRDpre:
2298 case AArch64::LDURQi:
2299 case AArch64::LDRQpre:
2300 case AArch64::LDURWi:
2301 case AArch64::LDRWpre:
2302 case AArch64::LDURXi:
2303 case AArch64::LDRXpre:
2304 case AArch64::LDRSWpre:
2305 case AArch64::LDURSWi:
2306 case AArch64::LDURHHi:
2307 case AArch64::LDURBBi:
2308 case AArch64::LDURSBWi:
2309 case AArch64::LDURSHWi:
2317 case AArch64::PRFMui:
return AArch64::PRFUMi;
2318 case AArch64::LDRXui:
return AArch64::LDURXi;
2319 case AArch64::LDRWui:
return AArch64::LDURWi;
2320 case AArch64::LDRBui:
return AArch64::LDURBi;
2321 case AArch64::LDRHui:
return AArch64::LDURHi;
2322 case AArch64::LDRSui:
return AArch64::LDURSi;
2323 case AArch64::LDRDui:
return AArch64::LDURDi;
2324 case AArch64::LDRQui:
return AArch64::LDURQi;
2325 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2326 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2327 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2328 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2329 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2330 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2331 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2332 case AArch64::STRXui:
return AArch64::STURXi;
2333 case AArch64::STRWui:
return AArch64::STURWi;
2334 case AArch64::STRBui:
return AArch64::STURBi;
2335 case AArch64::STRHui:
return AArch64::STURHi;
2336 case AArch64::STRSui:
return AArch64::STURSi;
2337 case AArch64::STRDui:
return AArch64::STURDi;
2338 case AArch64::STRQui:
return AArch64::STURQi;
2339 case AArch64::STRBBui:
return AArch64::STURBBi;
2340 case AArch64::STRHHui:
return AArch64::STURHHi;
2348 case AArch64::LDPXi:
2349 case AArch64::LDPDi:
2350 case AArch64::STPXi:
2351 case AArch64::STPDi:
2352 case AArch64::LDNPXi:
2353 case AArch64::LDNPDi:
2354 case AArch64::STNPXi:
2355 case AArch64::STNPDi:
2356 case AArch64::LDPQi:
2357 case AArch64::STPQi:
2358 case AArch64::LDNPQi:
2359 case AArch64::STNPQi:
2360 case AArch64::LDPWi:
2361 case AArch64::LDPSi:
2362 case AArch64::STPWi:
2363 case AArch64::STPSi:
2364 case AArch64::LDNPWi:
2365 case AArch64::LDNPSi:
2366 case AArch64::STNPWi:
2367 case AArch64::STNPSi:
2369 case AArch64::STGPi:
2371 case AArch64::LD1B_IMM:
2372 case AArch64::LD1B_H_IMM:
2373 case AArch64::LD1B_S_IMM:
2374 case AArch64::LD1B_D_IMM:
2375 case AArch64::LD1SB_H_IMM:
2376 case AArch64::LD1SB_S_IMM:
2377 case AArch64::LD1SB_D_IMM:
2378 case AArch64::LD1H_IMM:
2379 case AArch64::LD1H_S_IMM:
2380 case AArch64::LD1H_D_IMM:
2381 case AArch64::LD1SH_S_IMM:
2382 case AArch64::LD1SH_D_IMM:
2383 case AArch64::LD1W_IMM:
2384 case AArch64::LD1W_D_IMM:
2385 case AArch64::LD1SW_D_IMM:
2386 case AArch64::LD1D_IMM:
2388 case AArch64::LD2B_IMM:
2389 case AArch64::LD2H_IMM:
2390 case AArch64::LD2W_IMM:
2391 case AArch64::LD2D_IMM:
2392 case AArch64::LD3B_IMM:
2393 case AArch64::LD3H_IMM:
2394 case AArch64::LD3W_IMM:
2395 case AArch64::LD3D_IMM:
2396 case AArch64::LD4B_IMM:
2397 case AArch64::LD4H_IMM:
2398 case AArch64::LD4W_IMM:
2399 case AArch64::LD4D_IMM:
2401 case AArch64::ST1B_IMM:
2402 case AArch64::ST1B_H_IMM:
2403 case AArch64::ST1B_S_IMM:
2404 case AArch64::ST1B_D_IMM:
2405 case AArch64::ST1H_IMM:
2406 case AArch64::ST1H_S_IMM:
2407 case AArch64::ST1H_D_IMM:
2408 case AArch64::ST1W_IMM:
2409 case AArch64::ST1W_D_IMM:
2410 case AArch64::ST1D_IMM:
2412 case AArch64::ST2B_IMM:
2413 case AArch64::ST2H_IMM:
2414 case AArch64::ST2W_IMM:
2415 case AArch64::ST2D_IMM:
2416 case AArch64::ST3B_IMM:
2417 case AArch64::ST3H_IMM:
2418 case AArch64::ST3W_IMM:
2419 case AArch64::ST3D_IMM:
2420 case AArch64::ST4B_IMM:
2421 case AArch64::ST4H_IMM:
2422 case AArch64::ST4W_IMM:
2423 case AArch64::ST4D_IMM:
2425 case AArch64::LD1RB_IMM:
2426 case AArch64::LD1RB_H_IMM:
2427 case AArch64::LD1RB_S_IMM:
2428 case AArch64::LD1RB_D_IMM:
2429 case AArch64::LD1RSB_H_IMM:
2430 case AArch64::LD1RSB_S_IMM:
2431 case AArch64::LD1RSB_D_IMM:
2432 case AArch64::LD1RH_IMM:
2433 case AArch64::LD1RH_S_IMM:
2434 case AArch64::LD1RH_D_IMM:
2435 case AArch64::LD1RSH_S_IMM:
2436 case AArch64::LD1RSH_D_IMM:
2437 case AArch64::LD1RW_IMM:
2438 case AArch64::LD1RW_D_IMM:
2439 case AArch64::LD1RSW_IMM:
2440 case AArch64::LD1RD_IMM:
2442 case AArch64::LDNT1B_ZRI:
2443 case AArch64::LDNT1H_ZRI:
2444 case AArch64::LDNT1W_ZRI:
2445 case AArch64::LDNT1D_ZRI:
2446 case AArch64::STNT1B_ZRI:
2447 case AArch64::STNT1H_ZRI:
2448 case AArch64::STNT1W_ZRI:
2449 case AArch64::STNT1D_ZRI:
2451 case AArch64::LDNF1B_IMM:
2452 case AArch64::LDNF1B_H_IMM:
2453 case AArch64::LDNF1B_S_IMM:
2454 case AArch64::LDNF1B_D_IMM:
2455 case AArch64::LDNF1SB_H_IMM:
2456 case AArch64::LDNF1SB_S_IMM:
2457 case AArch64::LDNF1SB_D_IMM:
2458 case AArch64::LDNF1H_IMM:
2459 case AArch64::LDNF1H_S_IMM:
2460 case AArch64::LDNF1H_D_IMM:
2461 case AArch64::LDNF1SH_S_IMM:
2462 case AArch64::LDNF1SH_D_IMM:
2463 case AArch64::LDNF1W_IMM:
2464 case AArch64::LDNF1W_D_IMM:
2465 case AArch64::LDNF1SW_D_IMM:
2466 case AArch64::LDNF1D_IMM:
2470 case AArch64::LDR_PXI:
2471 case AArch64::STR_PXI:
2477 switch (
MI.getOpcode()) {
2481 case AArch64::STRSui:
2482 case AArch64::STRDui:
2483 case AArch64::STRQui:
2484 case AArch64::STRXui:
2485 case AArch64::STRWui:
2486 case AArch64::LDRSui:
2487 case AArch64::LDRDui:
2488 case AArch64::LDRQui:
2489 case AArch64::LDRXui:
2490 case AArch64::LDRWui:
2491 case AArch64::LDRSWui:
2493 case AArch64::STURSi:
2494 case AArch64::STRSpre:
2495 case AArch64::STURDi:
2496 case AArch64::STRDpre:
2497 case AArch64::STURQi:
2498 case AArch64::STRQpre:
2499 case AArch64::STURWi:
2500 case AArch64::STRWpre:
2501 case AArch64::STURXi:
2502 case AArch64::STRXpre:
2503 case AArch64::LDURSi:
2504 case AArch64::LDRSpre:
2505 case AArch64::LDURDi:
2506 case AArch64::LDRDpre:
2507 case AArch64::LDURQi:
2508 case AArch64::LDRQpre:
2509 case AArch64::LDURWi:
2510 case AArch64::LDRWpre:
2511 case AArch64::LDURXi:
2512 case AArch64::LDRXpre:
2513 case AArch64::LDURSWi:
2514 case AArch64::LDRSWpre:
2520 switch (
MI.getOpcode()) {
2523 "Unexpected instruction - was a new tail call opcode introduced?");
2525 case AArch64::TCRETURNdi:
2526 case AArch64::TCRETURNri:
2527 case AArch64::TCRETURNrix16x17:
2528 case AArch64::TCRETURNrix17:
2529 case AArch64::TCRETURNrinotx16:
2530 case AArch64::TCRETURNriALL:
2531 case AArch64::AUTH_TCRETURN:
2532 case AArch64::AUTH_TCRETURN_BTI:
2542 case AArch64::ADDWri:
2543 return AArch64::ADDSWri;
2544 case AArch64::ADDWrr:
2545 return AArch64::ADDSWrr;
2546 case AArch64::ADDWrs:
2547 return AArch64::ADDSWrs;
2548 case AArch64::ADDWrx:
2549 return AArch64::ADDSWrx;
2550 case AArch64::ANDWri:
2551 return AArch64::ANDSWri;
2552 case AArch64::ANDWrr:
2553 return AArch64::ANDSWrr;
2554 case AArch64::ANDWrs:
2555 return AArch64::ANDSWrs;
2556 case AArch64::BICWrr:
2557 return AArch64::BICSWrr;
2558 case AArch64::BICWrs:
2559 return AArch64::BICSWrs;
2560 case AArch64::SUBWri:
2561 return AArch64::SUBSWri;
2562 case AArch64::SUBWrr:
2563 return AArch64::SUBSWrr;
2564 case AArch64::SUBWrs:
2565 return AArch64::SUBSWrs;
2566 case AArch64::SUBWrx:
2567 return AArch64::SUBSWrx;
2569 case AArch64::ADDXri:
2570 return AArch64::ADDSXri;
2571 case AArch64::ADDXrr:
2572 return AArch64::ADDSXrr;
2573 case AArch64::ADDXrs:
2574 return AArch64::ADDSXrs;
2575 case AArch64::ADDXrx:
2576 return AArch64::ADDSXrx;
2577 case AArch64::ANDXri:
2578 return AArch64::ANDSXri;
2579 case AArch64::ANDXrr:
2580 return AArch64::ANDSXrr;
2581 case AArch64::ANDXrs:
2582 return AArch64::ANDSXrs;
2583 case AArch64::BICXrr:
2584 return AArch64::BICSXrr;
2585 case AArch64::BICXrs:
2586 return AArch64::BICSXrs;
2587 case AArch64::SUBXri:
2588 return AArch64::SUBSXri;
2589 case AArch64::SUBXrr:
2590 return AArch64::SUBSXrr;
2591 case AArch64::SUBXrs:
2592 return AArch64::SUBSXrs;
2593 case AArch64::SUBXrx:
2594 return AArch64::SUBSXrx;
2596 case AArch64::AND_PPzPP:
2597 return AArch64::ANDS_PPzPP;
2598 case AArch64::BIC_PPzPP:
2599 return AArch64::BICS_PPzPP;
2600 case AArch64::EOR_PPzPP:
2601 return AArch64::EORS_PPzPP;
2602 case AArch64::NAND_PPzPP:
2603 return AArch64::NANDS_PPzPP;
2604 case AArch64::NOR_PPzPP:
2605 return AArch64::NORS_PPzPP;
2606 case AArch64::ORN_PPzPP:
2607 return AArch64::ORNS_PPzPP;
2608 case AArch64::ORR_PPzPP:
2609 return AArch64::ORRS_PPzPP;
2610 case AArch64::BRKA_PPzP:
2611 return AArch64::BRKAS_PPzP;
2612 case AArch64::BRKPA_PPzPP:
2613 return AArch64::BRKPAS_PPzPP;
2614 case AArch64::BRKB_PPzP:
2615 return AArch64::BRKBS_PPzP;
2616 case AArch64::BRKPB_PPzPP:
2617 return AArch64::BRKPBS_PPzPP;
2618 case AArch64::BRKN_PPzP:
2619 return AArch64::BRKNS_PPzP;
2620 case AArch64::RDFFR_PPz:
2621 return AArch64::RDFFRS_PPz;
2622 case AArch64::PTRUE_B:
2623 return AArch64::PTRUES_B;
2634 if (
MI.hasOrderedMemoryRef())
2639 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2640 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2641 "Expected a reg or frame index operand.");
2645 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2647 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2660 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2661 Register BaseReg =
MI.getOperand(1).getReg();
2663 if (
MI.modifiesRegister(BaseReg,
TRI))
2676 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2678 MI.getMF()->getFunction().needsUnwindTableEntry();
2684 if (Subtarget.isPaired128Slow()) {
2685 switch (
MI.getOpcode()) {
2688 case AArch64::LDURQi:
2689 case AArch64::STURQi:
2690 case AArch64::LDRQui:
2691 case AArch64::STRQui:
2718std::optional<ExtAddrMode>
2723 bool OffsetIsScalable;
2724 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2725 return std::nullopt;
2728 return std::nullopt;
2743 int64_t OffsetScale = 1;
2748 case AArch64::LDURQi:
2749 case AArch64::STURQi:
2753 case AArch64::LDURDi:
2754 case AArch64::STURDi:
2755 case AArch64::LDURXi:
2756 case AArch64::STURXi:
2760 case AArch64::LDURWi:
2761 case AArch64::LDURSWi:
2762 case AArch64::STURWi:
2766 case AArch64::LDURHi:
2767 case AArch64::STURHi:
2768 case AArch64::LDURHHi:
2769 case AArch64::STURHHi:
2770 case AArch64::LDURSHXi:
2771 case AArch64::LDURSHWi:
2775 case AArch64::LDRBroX:
2776 case AArch64::LDRBBroX:
2777 case AArch64::LDRSBXroX:
2778 case AArch64::LDRSBWroX:
2779 case AArch64::STRBroX:
2780 case AArch64::STRBBroX:
2781 case AArch64::LDURBi:
2782 case AArch64::LDURBBi:
2783 case AArch64::LDURSBXi:
2784 case AArch64::LDURSBWi:
2785 case AArch64::STURBi:
2786 case AArch64::STURBBi:
2787 case AArch64::LDRBui:
2788 case AArch64::LDRBBui:
2789 case AArch64::LDRSBXui:
2790 case AArch64::LDRSBWui:
2791 case AArch64::STRBui:
2792 case AArch64::STRBBui:
2796 case AArch64::LDRQroX:
2797 case AArch64::STRQroX:
2798 case AArch64::LDRQui:
2799 case AArch64::STRQui:
2804 case AArch64::LDRDroX:
2805 case AArch64::STRDroX:
2806 case AArch64::LDRXroX:
2807 case AArch64::STRXroX:
2808 case AArch64::LDRDui:
2809 case AArch64::STRDui:
2810 case AArch64::LDRXui:
2811 case AArch64::STRXui:
2816 case AArch64::LDRWroX:
2817 case AArch64::LDRSWroX:
2818 case AArch64::STRWroX:
2819 case AArch64::LDRWui:
2820 case AArch64::LDRSWui:
2821 case AArch64::STRWui:
2826 case AArch64::LDRHroX:
2827 case AArch64::STRHroX:
2828 case AArch64::LDRHHroX:
2829 case AArch64::STRHHroX:
2830 case AArch64::LDRSHXroX:
2831 case AArch64::LDRSHWroX:
2832 case AArch64::LDRHui:
2833 case AArch64::STRHui:
2834 case AArch64::LDRHHui:
2835 case AArch64::STRHHui:
2836 case AArch64::LDRSHXui:
2837 case AArch64::LDRSHWui:
2845 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
2869 case AArch64::SBFMXri:
2882 AM.
Scale = OffsetScale;
2887 case TargetOpcode::SUBREG_TO_REG: {
2900 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
2904 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
2906 DefMI.getOperand(3).getImm() != 0)
2913 AM.
Scale = OffsetScale;
2924 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
2925 int64_t NewOffset) ->
bool {
2926 int64_t MinOffset, MaxOffset;
2943 return OldOffset < MinOffset || OldOffset > MaxOffset ||
2944 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
2946 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
2948 int64_t NewOffset = OldOffset + Disp;
2953 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
2963 auto canFoldAddRegIntoAddrMode =
2980 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
2981 Subtarget.isSTRQroSlow();
2990 case AArch64::ADDXri:
2996 return canFoldAddSubImmIntoAddrMode(Disp);
2998 case AArch64::SUBXri:
3004 return canFoldAddSubImmIntoAddrMode(-Disp);
3006 case AArch64::ADDXrs: {
3019 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3021 if (avoidSlowSTRQ(MemI))
3024 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3027 case AArch64::ADDXrr:
3035 if (!OptSize && avoidSlowSTRQ(MemI))
3037 return canFoldAddRegIntoAddrMode(1);
3039 case AArch64::ADDXrx:
3047 if (!OptSize && avoidSlowSTRQ(MemI))
3056 return canFoldAddRegIntoAddrMode(
3071 case AArch64::LDURQi:
3072 case AArch64::LDRQui:
3073 return AArch64::LDRQroX;
3074 case AArch64::STURQi:
3075 case AArch64::STRQui:
3076 return AArch64::STRQroX;
3077 case AArch64::LDURDi:
3078 case AArch64::LDRDui:
3079 return AArch64::LDRDroX;
3080 case AArch64::STURDi:
3081 case AArch64::STRDui:
3082 return AArch64::STRDroX;
3083 case AArch64::LDURXi:
3084 case AArch64::LDRXui:
3085 return AArch64::LDRXroX;
3086 case AArch64::STURXi:
3087 case AArch64::STRXui:
3088 return AArch64::STRXroX;
3089 case AArch64::LDURWi:
3090 case AArch64::LDRWui:
3091 return AArch64::LDRWroX;
3092 case AArch64::LDURSWi:
3093 case AArch64::LDRSWui:
3094 return AArch64::LDRSWroX;
3095 case AArch64::STURWi:
3096 case AArch64::STRWui:
3097 return AArch64::STRWroX;
3098 case AArch64::LDURHi:
3099 case AArch64::LDRHui:
3100 return AArch64::LDRHroX;
3101 case AArch64::STURHi:
3102 case AArch64::STRHui:
3103 return AArch64::STRHroX;
3104 case AArch64::LDURHHi:
3105 case AArch64::LDRHHui:
3106 return AArch64::LDRHHroX;
3107 case AArch64::STURHHi:
3108 case AArch64::STRHHui:
3109 return AArch64::STRHHroX;
3110 case AArch64::LDURSHXi:
3111 case AArch64::LDRSHXui:
3112 return AArch64::LDRSHXroX;
3113 case AArch64::LDURSHWi:
3114 case AArch64::LDRSHWui:
3115 return AArch64::LDRSHWroX;
3116 case AArch64::LDURBi:
3117 case AArch64::LDRBui:
3118 return AArch64::LDRBroX;
3119 case AArch64::LDURBBi:
3120 case AArch64::LDRBBui:
3121 return AArch64::LDRBBroX;
3122 case AArch64::LDURSBXi:
3123 case AArch64::LDRSBXui:
3124 return AArch64::LDRSBXroX;
3125 case AArch64::LDURSBWi:
3126 case AArch64::LDRSBWui:
3127 return AArch64::LDRSBWroX;
3128 case AArch64::STURBi:
3129 case AArch64::STRBui:
3130 return AArch64::STRBroX;
3131 case AArch64::STURBBi:
3132 case AArch64::STRBBui:
3133 return AArch64::STRBBroX;
3145 case AArch64::LDURQi:
3147 return AArch64::LDRQui;
3148 case AArch64::STURQi:
3150 return AArch64::STRQui;
3151 case AArch64::LDURDi:
3153 return AArch64::LDRDui;
3154 case AArch64::STURDi:
3156 return AArch64::STRDui;
3157 case AArch64::LDURXi:
3159 return AArch64::LDRXui;
3160 case AArch64::STURXi:
3162 return AArch64::STRXui;
3163 case AArch64::LDURWi:
3165 return AArch64::LDRWui;
3166 case AArch64::LDURSWi:
3168 return AArch64::LDRSWui;
3169 case AArch64::STURWi:
3171 return AArch64::STRWui;
3172 case AArch64::LDURHi:
3174 return AArch64::LDRHui;
3175 case AArch64::STURHi:
3177 return AArch64::STRHui;
3178 case AArch64::LDURHHi:
3180 return AArch64::LDRHHui;
3181 case AArch64::STURHHi:
3183 return AArch64::STRHHui;
3184 case AArch64::LDURSHXi:
3186 return AArch64::LDRSHXui;
3187 case AArch64::LDURSHWi:
3189 return AArch64::LDRSHWui;
3190 case AArch64::LDURBi:
3192 return AArch64::LDRBui;
3193 case AArch64::LDURBBi:
3195 return AArch64::LDRBBui;
3196 case AArch64::LDURSBXi:
3198 return AArch64::LDRSBXui;
3199 case AArch64::LDURSBWi:
3201 return AArch64::LDRSBWui;
3202 case AArch64::STURBi:
3204 return AArch64::STRBui;
3205 case AArch64::STURBBi:
3207 return AArch64::STRBBui;
3208 case AArch64::LDRQui:
3209 case AArch64::STRQui:
3212 case AArch64::LDRDui:
3213 case AArch64::STRDui:
3214 case AArch64::LDRXui:
3215 case AArch64::STRXui:
3218 case AArch64::LDRWui:
3219 case AArch64::LDRSWui:
3220 case AArch64::STRWui:
3223 case AArch64::LDRHui:
3224 case AArch64::STRHui:
3225 case AArch64::LDRHHui:
3226 case AArch64::STRHHui:
3227 case AArch64::LDRSHXui:
3228 case AArch64::LDRSHWui:
3231 case AArch64::LDRBui:
3232 case AArch64::LDRBBui:
3233 case AArch64::LDRSBXui:
3234 case AArch64::LDRSBWui:
3235 case AArch64::STRBui:
3236 case AArch64::STRBBui:
3250 case AArch64::LDURQi:
3251 case AArch64::STURQi:
3252 case AArch64::LDURDi:
3253 case AArch64::STURDi:
3254 case AArch64::LDURXi:
3255 case AArch64::STURXi:
3256 case AArch64::LDURWi:
3257 case AArch64::LDURSWi:
3258 case AArch64::STURWi:
3259 case AArch64::LDURHi:
3260 case AArch64::STURHi:
3261 case AArch64::LDURHHi:
3262 case AArch64::STURHHi:
3263 case AArch64::LDURSHXi:
3264 case AArch64::LDURSHWi:
3265 case AArch64::LDURBi:
3266 case AArch64::STURBi:
3267 case AArch64::LDURBBi:
3268 case AArch64::STURBBi:
3269 case AArch64::LDURSBWi:
3270 case AArch64::LDURSBXi:
3272 case AArch64::LDRQui:
3273 return AArch64::LDURQi;
3274 case AArch64::STRQui:
3275 return AArch64::STURQi;
3276 case AArch64::LDRDui:
3277 return AArch64::LDURDi;
3278 case AArch64::STRDui:
3279 return AArch64::STURDi;
3280 case AArch64::LDRXui:
3281 return AArch64::LDURXi;
3282 case AArch64::STRXui:
3283 return AArch64::STURXi;
3284 case AArch64::LDRWui:
3285 return AArch64::LDURWi;
3286 case AArch64::LDRSWui:
3287 return AArch64::LDURSWi;
3288 case AArch64::STRWui:
3289 return AArch64::STURWi;
3290 case AArch64::LDRHui:
3291 return AArch64::LDURHi;
3292 case AArch64::STRHui:
3293 return AArch64::STURHi;
3294 case AArch64::LDRHHui:
3295 return AArch64::LDURHHi;
3296 case AArch64::STRHHui:
3297 return AArch64::STURHHi;
3298 case AArch64::LDRSHXui:
3299 return AArch64::LDURSHXi;
3300 case AArch64::LDRSHWui:
3301 return AArch64::LDURSHWi;
3302 case AArch64::LDRBBui:
3303 return AArch64::LDURBBi;
3304 case AArch64::LDRBui:
3305 return AArch64::LDURBi;
3306 case AArch64::STRBBui:
3307 return AArch64::STURBBi;
3308 case AArch64::STRBui:
3309 return AArch64::STURBi;
3310 case AArch64::LDRSBWui:
3311 return AArch64::LDURSBWi;
3312 case AArch64::LDRSBXui:
3313 return AArch64::LDURSBXi;
3326 case AArch64::LDRQroX:
3327 case AArch64::LDURQi:
3328 case AArch64::LDRQui:
3329 return AArch64::LDRQroW;
3330 case AArch64::STRQroX:
3331 case AArch64::STURQi:
3332 case AArch64::STRQui:
3333 return AArch64::STRQroW;
3334 case AArch64::LDRDroX:
3335 case AArch64::LDURDi:
3336 case AArch64::LDRDui:
3337 return AArch64::LDRDroW;
3338 case AArch64::STRDroX:
3339 case AArch64::STURDi:
3340 case AArch64::STRDui:
3341 return AArch64::STRDroW;
3342 case AArch64::LDRXroX:
3343 case AArch64::LDURXi:
3344 case AArch64::LDRXui:
3345 return AArch64::LDRXroW;
3346 case AArch64::STRXroX:
3347 case AArch64::STURXi:
3348 case AArch64::STRXui:
3349 return AArch64::STRXroW;
3350 case AArch64::LDRWroX:
3351 case AArch64::LDURWi:
3352 case AArch64::LDRWui:
3353 return AArch64::LDRWroW;
3354 case AArch64::LDRSWroX:
3355 case AArch64::LDURSWi:
3356 case AArch64::LDRSWui:
3357 return AArch64::LDRSWroW;
3358 case AArch64::STRWroX:
3359 case AArch64::STURWi:
3360 case AArch64::STRWui:
3361 return AArch64::STRWroW;
3362 case AArch64::LDRHroX:
3363 case AArch64::LDURHi:
3364 case AArch64::LDRHui:
3365 return AArch64::LDRHroW;
3366 case AArch64::STRHroX:
3367 case AArch64::STURHi:
3368 case AArch64::STRHui:
3369 return AArch64::STRHroW;
3370 case AArch64::LDRHHroX:
3371 case AArch64::LDURHHi:
3372 case AArch64::LDRHHui:
3373 return AArch64::LDRHHroW;
3374 case AArch64::STRHHroX:
3375 case AArch64::STURHHi:
3376 case AArch64::STRHHui:
3377 return AArch64::STRHHroW;
3378 case AArch64::LDRSHXroX:
3379 case AArch64::LDURSHXi:
3380 case AArch64::LDRSHXui:
3381 return AArch64::LDRSHXroW;
3382 case AArch64::LDRSHWroX:
3383 case AArch64::LDURSHWi:
3384 case AArch64::LDRSHWui:
3385 return AArch64::LDRSHWroW;
3386 case AArch64::LDRBroX:
3387 case AArch64::LDURBi:
3388 case AArch64::LDRBui:
3389 return AArch64::LDRBroW;
3390 case AArch64::LDRBBroX:
3391 case AArch64::LDURBBi:
3392 case AArch64::LDRBBui:
3393 return AArch64::LDRBBroW;
3394 case AArch64::LDRSBXroX:
3395 case AArch64::LDURSBXi:
3396 case AArch64::LDRSBXui:
3397 return AArch64::LDRSBXroW;
3398 case AArch64::LDRSBWroX:
3399 case AArch64::LDURSBWi:
3400 case AArch64::LDRSBWui:
3401 return AArch64::LDRSBWroW;
3402 case AArch64::STRBroX:
3403 case AArch64::STURBi:
3404 case AArch64::STRBui:
3405 return AArch64::STRBroW;
3406 case AArch64::STRBBroX:
3407 case AArch64::STURBBi:
3408 case AArch64::STRBBui:
3409 return AArch64::STRBBroW;
3424 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3434 return B.getInstr();
3438 "Addressing mode not supported for folding");
3455 return B.getInstr();
3462 "Address offset can be a register or an immediate, but not both");
3464 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3469 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3483 return B.getInstr();
3487 "Function must not be called with an addressing mode it can't handle");
3492 bool &OffsetIsScalable,
TypeSize &Width,
3513 int64_t Dummy1, Dummy2;
3532 if (!BaseOp->
isReg() && !BaseOp->
isFI())
3542 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3547 TypeSize &Width, int64_t &MinOffset,
3548 int64_t &MaxOffset) {
3554 MinOffset = MaxOffset = 0;
3556 case AArch64::STRWpost:
3557 case AArch64::LDRWpost:
3563 case AArch64::LDURQi:
3564 case AArch64::STURQi:
3570 case AArch64::PRFUMi:
3571 case AArch64::LDURXi:
3572 case AArch64::LDURDi:
3573 case AArch64::LDAPURXi:
3574 case AArch64::STURXi:
3575 case AArch64::STURDi:
3576 case AArch64::STLURXi:
3582 case AArch64::LDURWi:
3583 case AArch64::LDURSi:
3584 case AArch64::LDURSWi:
3585 case AArch64::LDAPURi:
3586 case AArch64::LDAPURSWi:
3587 case AArch64::STURWi:
3588 case AArch64::STURSi:
3589 case AArch64::STLURWi:
3595 case AArch64::LDURHi:
3596 case AArch64::LDURHHi:
3597 case AArch64::LDURSHXi:
3598 case AArch64::LDURSHWi:
3599 case AArch64::LDAPURHi:
3600 case AArch64::LDAPURSHWi:
3601 case AArch64::LDAPURSHXi:
3602 case AArch64::STURHi:
3603 case AArch64::STURHHi:
3604 case AArch64::STLURHi:
3610 case AArch64::LDURBi:
3611 case AArch64::LDURBBi:
3612 case AArch64::LDURSBXi:
3613 case AArch64::LDURSBWi:
3614 case AArch64::LDAPURBi:
3615 case AArch64::LDAPURSBWi:
3616 case AArch64::LDAPURSBXi:
3617 case AArch64::STURBi:
3618 case AArch64::STURBBi:
3619 case AArch64::STLURBi:
3625 case AArch64::LDPQi:
3626 case AArch64::LDNPQi:
3627 case AArch64::STPQi:
3628 case AArch64::STNPQi:
3634 case AArch64::LDRQui:
3635 case AArch64::STRQui:
3641 case AArch64::LDPXi:
3642 case AArch64::LDPDi:
3643 case AArch64::LDNPXi:
3644 case AArch64::LDNPDi:
3645 case AArch64::STPXi:
3646 case AArch64::STPDi:
3647 case AArch64::STNPXi:
3648 case AArch64::STNPDi:
3654 case AArch64::PRFMui:
3655 case AArch64::LDRXui:
3656 case AArch64::LDRDui:
3657 case AArch64::STRXui:
3658 case AArch64::STRDui:
3664 case AArch64::StoreSwiftAsyncContext:
3671 case AArch64::LDPWi:
3672 case AArch64::LDPSi:
3673 case AArch64::LDNPWi:
3674 case AArch64::LDNPSi:
3675 case AArch64::STPWi:
3676 case AArch64::STPSi:
3677 case AArch64::STNPWi:
3678 case AArch64::STNPSi:
3684 case AArch64::LDRWui:
3685 case AArch64::LDRSui:
3686 case AArch64::LDRSWui:
3687 case AArch64::STRWui:
3688 case AArch64::STRSui:
3694 case AArch64::LDRHui:
3695 case AArch64::LDRHHui:
3696 case AArch64::LDRSHWui:
3697 case AArch64::LDRSHXui:
3698 case AArch64::STRHui:
3699 case AArch64::STRHHui:
3705 case AArch64::LDRBui:
3706 case AArch64::LDRBBui:
3707 case AArch64::LDRSBWui:
3708 case AArch64::LDRSBXui:
3709 case AArch64::STRBui:
3710 case AArch64::STRBBui:
3716 case AArch64::STPXpre:
3717 case AArch64::LDPXpost:
3718 case AArch64::STPDpre:
3719 case AArch64::LDPDpost:
3725 case AArch64::STPQpre:
3726 case AArch64::LDPQpost:
3732 case AArch64::STRXpre:
3733 case AArch64::STRDpre:
3734 case AArch64::LDRXpost:
3735 case AArch64::LDRDpost:
3741 case AArch64::STRQpre:
3742 case AArch64::LDRQpost:
3754 case AArch64::TAGPstack:
3764 case AArch64::STZGi:
3770 case AArch64::STR_ZZZZXI:
3771 case AArch64::LDR_ZZZZXI:
3777 case AArch64::STR_ZZZXI:
3778 case AArch64::LDR_ZZZXI:
3784 case AArch64::STR_ZZXI:
3785 case AArch64::LDR_ZZXI:
3791 case AArch64::LDR_PXI:
3792 case AArch64::STR_PXI:
3798 case AArch64::LDR_PPXI:
3799 case AArch64::STR_PPXI:
3805 case AArch64::LDR_ZXI:
3806 case AArch64::STR_ZXI:
3812 case AArch64::LD1B_IMM:
3813 case AArch64::LD1H_IMM:
3814 case AArch64::LD1W_IMM:
3815 case AArch64::LD1D_IMM:
3816 case AArch64::LDNT1B_ZRI:
3817 case AArch64::LDNT1H_ZRI:
3818 case AArch64::LDNT1W_ZRI:
3819 case AArch64::LDNT1D_ZRI:
3820 case AArch64::ST1B_IMM:
3821 case AArch64::ST1H_IMM:
3822 case AArch64::ST1W_IMM:
3823 case AArch64::ST1D_IMM:
3824 case AArch64::STNT1B_ZRI:
3825 case AArch64::STNT1H_ZRI:
3826 case AArch64::STNT1W_ZRI:
3827 case AArch64::STNT1D_ZRI:
3828 case AArch64::LDNF1B_IMM:
3829 case AArch64::LDNF1H_IMM:
3830 case AArch64::LDNF1W_IMM:
3831 case AArch64::LDNF1D_IMM:
3839 case AArch64::LD2B_IMM:
3840 case AArch64::LD2H_IMM:
3841 case AArch64::LD2W_IMM:
3842 case AArch64::LD2D_IMM:
3843 case AArch64::ST2B_IMM:
3844 case AArch64::ST2H_IMM:
3845 case AArch64::ST2W_IMM:
3846 case AArch64::ST2D_IMM:
3852 case AArch64::LD3B_IMM:
3853 case AArch64::LD3H_IMM:
3854 case AArch64::LD3W_IMM:
3855 case AArch64::LD3D_IMM:
3856 case AArch64::ST3B_IMM:
3857 case AArch64::ST3H_IMM:
3858 case AArch64::ST3W_IMM:
3859 case AArch64::ST3D_IMM:
3865 case AArch64::LD4B_IMM:
3866 case AArch64::LD4H_IMM:
3867 case AArch64::LD4W_IMM:
3868 case AArch64::LD4D_IMM:
3869 case AArch64::ST4B_IMM:
3870 case AArch64::ST4H_IMM:
3871 case AArch64::ST4W_IMM:
3872 case AArch64::ST4D_IMM:
3878 case AArch64::LD1B_H_IMM:
3879 case AArch64::LD1SB_H_IMM:
3880 case AArch64::LD1H_S_IMM:
3881 case AArch64::LD1SH_S_IMM:
3882 case AArch64::LD1W_D_IMM:
3883 case AArch64::LD1SW_D_IMM:
3884 case AArch64::ST1B_H_IMM:
3885 case AArch64::ST1H_S_IMM:
3886 case AArch64::ST1W_D_IMM:
3887 case AArch64::LDNF1B_H_IMM:
3888 case AArch64::LDNF1SB_H_IMM:
3889 case AArch64::LDNF1H_S_IMM:
3890 case AArch64::LDNF1SH_S_IMM:
3891 case AArch64::LDNF1W_D_IMM:
3892 case AArch64::LDNF1SW_D_IMM:
3900 case AArch64::LD1B_S_IMM:
3901 case AArch64::LD1SB_S_IMM:
3902 case AArch64::LD1H_D_IMM:
3903 case AArch64::LD1SH_D_IMM:
3904 case AArch64::ST1B_S_IMM:
3905 case AArch64::ST1H_D_IMM:
3906 case AArch64::LDNF1B_S_IMM:
3907 case AArch64::LDNF1SB_S_IMM:
3908 case AArch64::LDNF1H_D_IMM:
3909 case AArch64::LDNF1SH_D_IMM:
3917 case AArch64::LD1B_D_IMM:
3918 case AArch64::LD1SB_D_IMM:
3919 case AArch64::ST1B_D_IMM:
3920 case AArch64::LDNF1B_D_IMM:
3921 case AArch64::LDNF1SB_D_IMM:
3929 case AArch64::ST2Gi:
3930 case AArch64::STZ2Gi:
3936 case AArch64::STGPi:
3942 case AArch64::LD1RB_IMM:
3943 case AArch64::LD1RB_H_IMM:
3944 case AArch64::LD1RB_S_IMM:
3945 case AArch64::LD1RB_D_IMM:
3946 case AArch64::LD1RSB_H_IMM:
3947 case AArch64::LD1RSB_S_IMM:
3948 case AArch64::LD1RSB_D_IMM:
3954 case AArch64::LD1RH_IMM:
3955 case AArch64::LD1RH_S_IMM:
3956 case AArch64::LD1RH_D_IMM:
3957 case AArch64::LD1RSH_S_IMM:
3958 case AArch64::LD1RSH_D_IMM:
3964 case AArch64::LD1RW_IMM:
3965 case AArch64::LD1RW_D_IMM:
3966 case AArch64::LD1RSW_IMM:
3972 case AArch64::LD1RD_IMM:
3988 case AArch64::LDRBBui:
3989 case AArch64::LDURBBi:
3990 case AArch64::LDRSBWui:
3991 case AArch64::LDURSBWi:
3992 case AArch64::STRBBui:
3993 case AArch64::STURBBi:
3995 case AArch64::LDRHHui:
3996 case AArch64::LDURHHi:
3997 case AArch64::LDRSHWui:
3998 case AArch64::LDURSHWi:
3999 case AArch64::STRHHui:
4000 case AArch64::STURHHi:
4002 case AArch64::LDRSui:
4003 case AArch64::LDURSi:
4004 case AArch64::LDRSpre:
4005 case AArch64::LDRSWui:
4006 case AArch64::LDURSWi:
4007 case AArch64::LDRSWpre:
4008 case AArch64::LDRWpre:
4009 case AArch64::LDRWui:
4010 case AArch64::LDURWi:
4011 case AArch64::STRSui:
4012 case AArch64::STURSi:
4013 case AArch64::STRSpre:
4014 case AArch64::STRWui:
4015 case AArch64::STURWi:
4016 case AArch64::STRWpre:
4017 case AArch64::LDPSi:
4018 case AArch64::LDPSWi:
4019 case AArch64::LDPWi:
4020 case AArch64::STPSi:
4021 case AArch64::STPWi:
4023 case AArch64::LDRDui:
4024 case AArch64::LDURDi:
4025 case AArch64::LDRDpre:
4026 case AArch64::LDRXui:
4027 case AArch64::LDURXi:
4028 case AArch64::LDRXpre:
4029 case AArch64::STRDui:
4030 case AArch64::STURDi:
4031 case AArch64::STRDpre:
4032 case AArch64::STRXui:
4033 case AArch64::STURXi:
4034 case AArch64::STRXpre:
4035 case AArch64::LDPDi:
4036 case AArch64::LDPXi:
4037 case AArch64::STPDi:
4038 case AArch64::STPXi:
4040 case AArch64::LDRQui:
4041 case AArch64::LDURQi:
4042 case AArch64::STRQui:
4043 case AArch64::STURQi:
4044 case AArch64::STRQpre:
4045 case AArch64::LDPQi:
4046 case AArch64::LDRQpre:
4047 case AArch64::STPQi:
4049 case AArch64::STZGi:
4050 case AArch64::ST2Gi:
4051 case AArch64::STZ2Gi:
4052 case AArch64::STGPi:
4058 switch (
MI.getOpcode()) {
4061 case AArch64::LDRWpre:
4062 case AArch64::LDRXpre:
4063 case AArch64::LDRSWpre:
4064 case AArch64::LDRSpre:
4065 case AArch64::LDRDpre:
4066 case AArch64::LDRQpre:
4072 switch (
MI.getOpcode()) {
4075 case AArch64::STRWpre:
4076 case AArch64::STRXpre:
4077 case AArch64::STRSpre:
4078 case AArch64::STRDpre:
4079 case AArch64::STRQpre:
4089 switch (
MI.getOpcode()) {
4092 case AArch64::LDPSi:
4093 case AArch64::LDPSWi:
4094 case AArch64::LDPDi:
4095 case AArch64::LDPQi:
4096 case AArch64::LDPWi:
4097 case AArch64::LDPXi:
4098 case AArch64::STPSi:
4099 case AArch64::STPDi:
4100 case AArch64::STPQi:
4101 case AArch64::STPWi:
4102 case AArch64::STPXi:
4103 case AArch64::STGPi:
4112 return MI.getOperand(
Idx);
4120 return MI.getOperand(
Idx);
4125 if (
MI.getParent() ==
nullptr)
4135 auto Reg =
Op.getReg();
4136 if (Reg.isPhysical())
4137 return AArch64::FPR16RegClass.
contains(Reg);
4139 return TRC == &AArch64::FPR16RegClass ||
4140 TRC == &AArch64::FPR16_loRegClass;
4149 auto Reg =
Op.getReg();
4150 if (Reg.isPhysical())
4151 return AArch64::FPR128RegClass.
contains(Reg);
4153 return TRC == &AArch64::FPR128RegClass ||
4154 TRC == &AArch64::FPR128_loRegClass;
4160 switch (
MI.getOpcode()) {
4163 case AArch64::PACIASP:
4164 case AArch64::PACIBSP:
4167 case AArch64::PAUTH_PROLOGUE:
4170 case AArch64::HINT: {
4171 unsigned Imm =
MI.getOperand(0).getImm();
4173 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4176 if (Imm == 25 || Imm == 27)
4189 auto Reg =
Op.getReg();
4190 if (Reg.isPhysical())
4191 return AArch64::FPR128RegClass.
contains(Reg) ||
4192 AArch64::FPR64RegClass.contains(Reg) ||
4193 AArch64::FPR32RegClass.contains(Reg) ||
4194 AArch64::FPR16RegClass.contains(Reg) ||
4195 AArch64::FPR8RegClass.contains(Reg);
4198 return TRC == &AArch64::FPR128RegClass ||
4199 TRC == &AArch64::FPR128_loRegClass ||
4200 TRC == &AArch64::FPR64RegClass ||
4201 TRC == &AArch64::FPR64_loRegClass ||
4202 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4203 TRC == &AArch64::FPR8RegClass;
4225 if (FirstOpc == SecondOpc)
4231 case AArch64::STRSui:
4232 case AArch64::STURSi:
4233 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4234 case AArch64::STRDui:
4235 case AArch64::STURDi:
4236 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4237 case AArch64::STRQui:
4238 case AArch64::STURQi:
4239 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4240 case AArch64::STRWui:
4241 case AArch64::STURWi:
4242 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4243 case AArch64::STRXui:
4244 case AArch64::STURXi:
4245 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4246 case AArch64::LDRSui:
4247 case AArch64::LDURSi:
4248 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4249 case AArch64::LDRDui:
4250 case AArch64::LDURDi:
4251 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4252 case AArch64::LDRQui:
4253 case AArch64::LDURQi:
4254 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4255 case AArch64::LDRWui:
4256 case AArch64::LDURWi:
4257 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4258 case AArch64::LDRSWui:
4259 case AArch64::LDURSWi:
4260 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4261 case AArch64::LDRXui:
4262 case AArch64::LDURXi:
4263 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4270 int64_t Offset1,
unsigned Opcode1,
int FI2,
4271 int64_t Offset2,
unsigned Opcode2) {
4277 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4280 if (ObjectOffset1 % Scale1 != 0)
4282 ObjectOffset1 /= Scale1;
4284 if (ObjectOffset2 % Scale2 != 0)
4286 ObjectOffset2 /= Scale2;
4287 ObjectOffset1 += Offset1;
4288 ObjectOffset2 += Offset2;
4289 return ObjectOffset1 + 1 == ObjectOffset2;
4301 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4302 unsigned NumBytes)
const {
4312 "Only base registers and frame indices are supported.");
4319 if (ClusterSize > 2)
4326 unsigned FirstOpc = FirstLdSt.
getOpcode();
4327 unsigned SecondOpc = SecondLdSt.
getOpcode();
4347 if (Offset1 > 63 || Offset1 < -64)
4352 if (BaseOp1.
isFI()) {
4354 "Caller should have ordered offsets.");
4359 BaseOp2.
getIndex(), Offset2, SecondOpc);
4362 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4364 return Offset1 + 1 == Offset2;
4368 unsigned Reg,
unsigned SubIdx,
4372 return MIB.
addReg(Reg, State);
4375 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4376 return MIB.
addReg(Reg, State, SubIdx);
4383 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4392 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4394 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4395 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4396 unsigned NumRegs = Indices.
size();
4398 int SubReg = 0,
End = NumRegs, Incr = 1;
4416 unsigned SrcReg,
bool KillSrc,
4417 unsigned Opcode,
unsigned ZeroReg,
4420 unsigned NumRegs = Indices.
size();
4423 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4424 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4425 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4426 "GPR reg sequences should not be able to overlap");
4442 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4443 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4446 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4448 if (Subtarget.hasZeroCycleRegMove()) {
4451 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4453 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4469 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4474 if (Subtarget.hasZeroCycleRegMove()) {
4477 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4479 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4499 if (AArch64::PPRRegClass.
contains(DestReg) &&
4500 AArch64::PPRRegClass.
contains(SrcReg)) {
4502 "Unexpected SVE register.");
4512 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4513 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4514 if (DestIsPNR || SrcIsPNR) {
4516 return (R - AArch64::PN0) + AArch64::P0;
4518 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4519 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4521 if (PPRSrcReg != PPRDestReg) {
4533 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4534 AArch64::ZPRRegClass.
contains(SrcReg)) {
4536 "Unexpected SVE register.");
4544 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
4545 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
4546 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
4547 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
4549 "Unexpected SVE register.");
4550 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
4557 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
4558 AArch64::ZPR3RegClass.
contains(SrcReg)) {
4560 "Unexpected SVE register.");
4561 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4569 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
4570 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
4571 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
4572 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
4574 "Unexpected SVE register.");
4575 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4576 AArch64::zsub2, AArch64::zsub3};
4582 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
4583 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
4584 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
4590 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
4604 if (AArch64::DDDDRegClass.
contains(DestReg) &&
4605 AArch64::DDDDRegClass.
contains(SrcReg)) {
4606 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4607 AArch64::dsub2, AArch64::dsub3};
4614 if (AArch64::DDDRegClass.
contains(DestReg) &&
4615 AArch64::DDDRegClass.
contains(SrcReg)) {
4616 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4624 if (AArch64::DDRegClass.
contains(DestReg) &&
4625 AArch64::DDRegClass.
contains(SrcReg)) {
4626 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
4633 if (AArch64::QQQQRegClass.
contains(DestReg) &&
4634 AArch64::QQQQRegClass.
contains(SrcReg)) {
4635 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4636 AArch64::qsub2, AArch64::qsub3};
4643 if (AArch64::QQQRegClass.
contains(DestReg) &&
4644 AArch64::QQQRegClass.
contains(SrcReg)) {
4645 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4653 if (AArch64::QQRegClass.
contains(DestReg) &&
4654 AArch64::QQRegClass.
contains(SrcReg)) {
4655 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
4661 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
4662 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
4663 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
4665 AArch64::XZR, Indices);
4669 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
4670 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
4671 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
4673 AArch64::WZR, Indices);
4677 if (AArch64::FPR128RegClass.
contains(DestReg) &&
4678 AArch64::FPR128RegClass.
contains(SrcReg)) {
4683 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
4684 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
4704 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4705 AArch64::FPR64RegClass.
contains(SrcReg)) {
4711 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4712 AArch64::FPR32RegClass.
contains(SrcReg)) {
4718 if (AArch64::FPR16RegClass.
contains(DestReg) &&
4719 AArch64::FPR16RegClass.
contains(SrcReg)) {
4721 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
4723 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
4729 if (AArch64::FPR8RegClass.
contains(DestReg) &&
4730 AArch64::FPR8RegClass.
contains(SrcReg)) {
4732 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
4734 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
4741 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4742 AArch64::GPR64RegClass.
contains(SrcReg)) {
4747 if (AArch64::GPR64RegClass.
contains(DestReg) &&
4748 AArch64::FPR64RegClass.
contains(SrcReg)) {
4754 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4755 AArch64::GPR32RegClass.
contains(SrcReg)) {
4760 if (AArch64::GPR32RegClass.
contains(DestReg) &&
4761 AArch64::FPR32RegClass.
contains(SrcReg)) {
4767 if (DestReg == AArch64::NZCV) {
4768 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
4770 .
addImm(AArch64SysReg::NZCV)
4776 if (SrcReg == AArch64::NZCV) {
4777 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
4779 .
addImm(AArch64SysReg::NZCV)
4786 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
4787 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
4797 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
4802 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
4804 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
4817 Register SrcReg,
bool isKill,
int FI,
4832 switch (
TRI->getSpillSize(*RC)) {
4834 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4835 Opc = AArch64::STRBui;
4838 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4839 Opc = AArch64::STRHui;
4840 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
4841 AArch64::PPRRegClass.hasSubClassEq(RC)) {
4843 "Unexpected register store without SVE store instructions");
4844 Opc = AArch64::STR_PXI;
4850 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4851 Opc = AArch64::STRWui;
4855 assert(SrcReg != AArch64::WSP);
4856 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4857 Opc = AArch64::STRSui;
4858 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
4859 Opc = AArch64::STR_PPXI;
4864 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4865 Opc = AArch64::STRXui;
4869 assert(SrcReg != AArch64::SP);
4870 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4871 Opc = AArch64::STRDui;
4872 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4874 get(AArch64::STPWi), SrcReg, isKill,
4875 AArch64::sube32, AArch64::subo32, FI, MMO);
4880 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4881 Opc = AArch64::STRQui;
4882 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4883 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4884 Opc = AArch64::ST1Twov1d;
4886 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4888 get(AArch64::STPXi), SrcReg, isKill,
4889 AArch64::sube64, AArch64::subo64, FI, MMO);
4891 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4893 "Unexpected register store without SVE store instructions");
4894 Opc = AArch64::STR_ZXI;
4899 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4900 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4901 Opc = AArch64::ST1Threev1d;
4906 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4907 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4908 Opc = AArch64::ST1Fourv1d;
4910 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4911 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4912 Opc = AArch64::ST1Twov2d;
4914 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
4915 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4917 "Unexpected register store without SVE store instructions");
4918 Opc = AArch64::STR_ZZXI;
4923 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4924 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4925 Opc = AArch64::ST1Threev2d;
4927 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4929 "Unexpected register store without SVE store instructions");
4930 Opc = AArch64::STR_ZZZXI;
4935 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4936 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4937 Opc = AArch64::ST1Fourv2d;
4939 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
4940 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4942 "Unexpected register store without SVE store instructions");
4943 Opc = AArch64::STR_ZZZZXI;
4948 assert(Opc &&
"Unknown register class");
4959 MI.addMemOperand(MMO);
4966 Register DestReg,
unsigned SubIdx0,
4967 unsigned SubIdx1,
int FI,
4971 bool IsUndef =
true;
4973 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
4975 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
5004 switch (
TRI->getSpillSize(*RC)) {
5006 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
5007 Opc = AArch64::LDRBui;
5010 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
5011 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
5012 Opc = AArch64::LDRHui;
5013 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5015 "Unexpected register load without SVE load instructions");
5018 Opc = AArch64::LDR_PXI;
5024 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5025 Opc = AArch64::LDRWui;
5029 assert(DestReg != AArch64::WSP);
5030 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5031 Opc = AArch64::LDRSui;
5032 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5033 Opc = AArch64::LDR_PPXI;
5038 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5039 Opc = AArch64::LDRXui;
5043 assert(DestReg != AArch64::SP);
5044 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5045 Opc = AArch64::LDRDui;
5046 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5048 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5049 AArch64::subo32, FI, MMO);
5054 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5055 Opc = AArch64::LDRQui;
5056 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5057 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5058 Opc = AArch64::LD1Twov1d;
5060 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5062 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5063 AArch64::subo64, FI, MMO);
5065 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5067 "Unexpected register load without SVE load instructions");
5068 Opc = AArch64::LDR_ZXI;
5073 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5074 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5075 Opc = AArch64::LD1Threev1d;
5080 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5081 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5082 Opc = AArch64::LD1Fourv1d;
5084 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5085 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5086 Opc = AArch64::LD1Twov2d;
5088 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5089 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5091 "Unexpected register load without SVE load instructions");
5092 Opc = AArch64::LDR_ZZXI;
5097 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5098 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5099 Opc = AArch64::LD1Threev2d;
5101 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5103 "Unexpected register load without SVE load instructions");
5104 Opc = AArch64::LDR_ZZZXI;
5109 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5110 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5111 Opc = AArch64::LD1Fourv2d;
5113 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5114 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5116 "Unexpected register load without SVE load instructions");
5117 Opc = AArch64::LDR_ZZZZXI;
5123 assert(Opc &&
"Unknown register class");
5133 MI.addMemOperand(MMO);
5144 UseMI.getIterator()),
5146 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5147 I.readsRegister(AArch64::NZCV, TRI);
5156 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5163 ByteSized =
Offset.getFixed();
5164 VGSized =
Offset.getScalable() / 2;
5172 int64_t &NumDataVectors) {
5176 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5178 NumBytes =
Offset.getFixed();
5180 NumPredicateVectors =
Offset.getScalable() / 2;
5185 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5186 NumPredicateVectors > 62) {
5187 NumDataVectors = NumPredicateVectors / 8;
5188 NumPredicateVectors -= NumDataVectors * 8;
5195 int NumVGScaledBytes,
unsigned VG,
5202 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5203 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5206 if (NumVGScaledBytes) {
5207 Expr.
push_back((uint8_t)dwarf::DW_OP_consts);
5210 Expr.
push_back((uint8_t)dwarf::DW_OP_bregx);
5214 Expr.
push_back((uint8_t)dwarf::DW_OP_mul);
5215 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5217 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5218 << std::abs(NumVGScaledBytes) <<
" * VG";
5227 int64_t NumBytes, NumVGScaledBytes;
5230 std::string CommentBuffer;
5233 if (Reg == AArch64::SP)
5235 else if (Reg == AArch64::FP)
5242 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5243 Expr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
5246 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5250 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
5259 unsigned FrameReg,
unsigned Reg,
5261 bool LastAdjustmentWasScalable) {
5262 if (
Offset.getScalable())
5265 if (FrameReg == Reg && !LastAdjustmentWasScalable)
5268 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5275 int64_t NumBytes, NumVGScaledBytes;
5277 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
5279 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5282 if (!NumVGScaledBytes)
5285 std::string CommentBuffer;
5292 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5296 CfaExpr.
push_back(dwarf::DW_CFA_expression);
5312 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
5315 bool *HasWinCFI,
bool EmitCFAOffset,
5318 unsigned MaxEncoding, ShiftSize;
5320 case AArch64::ADDXri:
5321 case AArch64::ADDSXri:
5322 case AArch64::SUBXri:
5323 case AArch64::SUBSXri:
5324 MaxEncoding = 0xfff;
5327 case AArch64::ADDVL_XXI:
5328 case AArch64::ADDPL_XXI:
5329 case AArch64::ADDSVL_XXI:
5330 case AArch64::ADDSPL_XXI:
5345 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
5347 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
5361 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
5363 if (TmpReg == AArch64::XZR)
5365 &AArch64::GPR64RegClass);
5367 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
5368 unsigned LocalShiftSize = 0;
5369 if (ThisVal > MaxEncoding) {
5370 ThisVal = ThisVal >> ShiftSize;
5371 LocalShiftSize = ShiftSize;
5373 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
5374 "Encoding cannot handle value that big");
5376 Offset -= ThisVal << LocalShiftSize;
5381 .
addImm(Sign * (
int)ThisVal);
5391 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
5392 CFAOffset += Change;
5394 CFAOffset -= Change;
5395 if (EmitCFAOffset && DestReg == TmpReg) {
5408 assert(Sign == 1 &&
"SEH directives should always have a positive sign");
5409 int Imm = (int)(ThisVal << LocalShiftSize);
5410 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
5411 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
5420 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
5421 "emit a single SEH directive");
5422 }
else if (DestReg == AArch64::SP) {
5425 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
5438 unsigned DestReg,
unsigned SrcReg,
5441 bool NeedsWinCFI,
bool *HasWinCFI,
5443 unsigned FrameReg) {
5450 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
5452 int64_t Bytes, NumPredicateVectors, NumDataVectors;
5454 Offset, Bytes, NumPredicateVectors, NumDataVectors);
5457 if (Bytes || (!
Offset && SrcReg != DestReg)) {
5458 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
5459 "SP increment/decrement not 8-byte aligned");
5460 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
5463 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
5466 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
5468 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
5475 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
5476 "SetNZCV not supported with SVE vectors");
5477 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
5478 "WinCFI not supported with SVE vectors");
5480 if (NumDataVectors) {
5482 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
5483 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5484 CFAOffset, FrameReg);
5489 if (NumPredicateVectors) {
5490 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
5492 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
5493 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5494 CFAOffset, FrameReg);
5512 if (
MI.isFullCopy()) {
5515 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
5519 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
5524 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
5552 if (
MI.isCopy() && Ops.
size() == 1 &&
5554 (Ops[0] == 0 || Ops[0] == 1)) {
5555 bool IsSpill = Ops[0] == 0;
5556 bool IsFill = !IsSpill;
5568 :
TRI.getMinimalPhysRegClass(Reg);
5574 "Mismatched register size in non subreg COPY");
5581 return &*--InsertPt;
5593 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
5596 "Unexpected subreg on physical register");
5598 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
5600 return &*--InsertPt;
5618 case AArch64::sub_32:
5619 FillRC = &AArch64::GPR32RegClass;
5622 FillRC = &AArch64::FPR32RegClass;
5625 FillRC = &AArch64::FPR64RegClass;
5631 TRI.getRegSizeInBits(*FillRC) &&
5632 "Mismatched regclass size on folded subreg COPY");
5651 bool *OutUseUnscaledOp,
5652 unsigned *OutUnscaledOp,
5653 int64_t *EmittableOffset) {
5655 if (EmittableOffset)
5656 *EmittableOffset = 0;
5657 if (OutUseUnscaledOp)
5658 *OutUseUnscaledOp =
false;
5664 switch (
MI.getOpcode()) {
5667 case AArch64::LD1Rv1d:
5668 case AArch64::LD1Rv2s:
5669 case AArch64::LD1Rv2d:
5670 case AArch64::LD1Rv4h:
5671 case AArch64::LD1Rv4s:
5672 case AArch64::LD1Rv8b:
5673 case AArch64::LD1Rv8h:
5674 case AArch64::LD1Rv16b:
5675 case AArch64::LD1Twov2d:
5676 case AArch64::LD1Threev2d:
5677 case AArch64::LD1Fourv2d:
5678 case AArch64::LD1Twov1d:
5679 case AArch64::LD1Threev1d:
5680 case AArch64::LD1Fourv1d:
5681 case AArch64::ST1Twov2d:
5682 case AArch64::ST1Threev2d:
5683 case AArch64::ST1Fourv2d:
5684 case AArch64::ST1Twov1d:
5685 case AArch64::ST1Threev1d:
5686 case AArch64::ST1Fourv1d:
5687 case AArch64::ST1i8:
5688 case AArch64::ST1i16:
5689 case AArch64::ST1i32:
5690 case AArch64::ST1i64:
5692 case AArch64::IRGstack:
5693 case AArch64::STGloop:
5694 case AArch64::STZGloop:
5699 TypeSize ScaleValue(0U,
false), Width(0U,
false);
5700 int64_t MinOff, MaxOff;
5706 bool IsMulVL = ScaleValue.isScalable();
5707 unsigned Scale = ScaleValue.getKnownMinValue();
5717 std::optional<unsigned> UnscaledOp =
5719 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
5720 if (useUnscaledOp &&
5725 Scale = ScaleValue.getKnownMinValue();
5726 assert(IsMulVL == ScaleValue.isScalable() &&
5727 "Unscaled opcode has different value for scalable");
5729 int64_t Remainder =
Offset % Scale;
5730 assert(!(Remainder && useUnscaledOp) &&
5731 "Cannot have remainder when using unscaled op");
5733 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
5734 int64_t NewOffset =
Offset / Scale;
5735 if (MinOff <= NewOffset && NewOffset <= MaxOff)
5738 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
5742 if (EmittableOffset)
5743 *EmittableOffset = NewOffset;
5744 if (OutUseUnscaledOp)
5745 *OutUseUnscaledOp = useUnscaledOp;
5746 if (OutUnscaledOp && UnscaledOp)
5747 *OutUnscaledOp = *UnscaledOp;
5760 unsigned Opcode =
MI.getOpcode();
5761 unsigned ImmIdx = FrameRegIdx + 1;
5763 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
5768 MI.eraseFromParent();
5774 unsigned UnscaledOp;
5777 &UnscaledOp, &NewOffset);
5781 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
5783 MI.setDesc(
TII->get(UnscaledOp));
5785 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
5808 case AArch64::ADDSWrr:
5809 case AArch64::ADDSWri:
5810 case AArch64::ADDSXrr:
5811 case AArch64::ADDSXri:
5812 case AArch64::SUBSWrr:
5813 case AArch64::SUBSXrr:
5815 case AArch64::SUBSWri:
5816 case AArch64::SUBSXri:
5827 case AArch64::ADDWrr:
5828 case AArch64::ADDWri:
5829 case AArch64::SUBWrr:
5830 case AArch64::ADDSWrr:
5831 case AArch64::ADDSWri:
5832 case AArch64::SUBSWrr:
5834 case AArch64::SUBWri:
5835 case AArch64::SUBSWri:
5846 case AArch64::ADDXrr:
5847 case AArch64::ADDXri:
5848 case AArch64::SUBXrr:
5849 case AArch64::ADDSXrr:
5850 case AArch64::ADDSXri:
5851 case AArch64::SUBSXrr:
5853 case AArch64::SUBXri:
5854 case AArch64::SUBSXri:
5855 case AArch64::ADDv8i8:
5856 case AArch64::ADDv16i8:
5857 case AArch64::ADDv4i16:
5858 case AArch64::ADDv8i16:
5859 case AArch64::ADDv2i32:
5860 case AArch64::ADDv4i32:
5861 case AArch64::SUBv8i8:
5862 case AArch64::SUBv16i8:
5863 case AArch64::SUBv4i16:
5864 case AArch64::SUBv8i16:
5865 case AArch64::SUBv2i32:
5866 case AArch64::SUBv4i32:
5879 case AArch64::FADDHrr:
5880 case AArch64::FADDSrr:
5881 case AArch64::FADDDrr:
5882 case AArch64::FADDv4f16:
5883 case AArch64::FADDv8f16:
5884 case AArch64::FADDv2f32:
5885 case AArch64::FADDv2f64:
5886 case AArch64::FADDv4f32:
5887 case AArch64::FSUBHrr:
5888 case AArch64::FSUBSrr:
5889 case AArch64::FSUBDrr:
5890 case AArch64::FSUBv4f16:
5891 case AArch64::FSUBv8f16:
5892 case AArch64::FSUBv2f32:
5893 case AArch64::FSUBv2f64:
5894 case AArch64::FSUBv4f32:
5898 return Options.UnsafeFPMath ||
5915 unsigned CombineOpc,
unsigned ZeroReg = 0,
5916 bool CheckZeroReg =
false) {
5923 if (!
MI ||
MI->getParent() != &
MBB || (
unsigned)
MI->getOpcode() != CombineOpc)
5926 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
5930 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
5931 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
5932 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
5934 if (
MI->getOperand(3).getReg() != ZeroReg)
5939 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
5948 unsigned MulOpc,
unsigned ZeroReg) {
5964 bool Invert)
const {
5970 case AArch64::FADDHrr:
5971 case AArch64::FADDSrr:
5972 case AArch64::FADDDrr:
5973 case AArch64::FMULHrr:
5974 case AArch64::FMULSrr:
5975 case AArch64::FMULDrr:
5976 case AArch64::FMULX16:
5977 case AArch64::FMULX32:
5978 case AArch64::FMULX64:
5980 case AArch64::FADDv4f16:
5981 case AArch64::FADDv8f16:
5982 case AArch64::FADDv2f32:
5983 case AArch64::FADDv4f32:
5984 case AArch64::FADDv2f64:
5985 case AArch64::FMULv4f16:
5986 case AArch64::FMULv8f16:
5987 case AArch64::FMULv2f32:
5988 case AArch64::FMULv4f32:
5989 case AArch64::FMULv2f64:
5990 case AArch64::FMULXv4f16:
5991 case AArch64::FMULXv8f16:
5992 case AArch64::FMULXv2f32:
5993 case AArch64::FMULXv4f32:
5994 case AArch64::FMULXv2f64:
5998 case AArch64::FADD_ZZZ_H:
5999 case AArch64::FADD_ZZZ_S:
6000 case AArch64::FADD_ZZZ_D:
6001 case AArch64::FMUL_ZZZ_H:
6002 case AArch64::FMUL_ZZZ_S:
6003 case AArch64::FMUL_ZZZ_D:
6015 case AArch64::ADDWrr:
6016 case AArch64::ADDXrr:
6017 case AArch64::ANDWrr:
6018 case AArch64::ANDXrr:
6019 case AArch64::ORRWrr:
6020 case AArch64::ORRXrr:
6021 case AArch64::EORWrr:
6022 case AArch64::EORXrr:
6023 case AArch64::EONWrr:
6024 case AArch64::EONXrr:
6028 case AArch64::ADDv8i8:
6029 case AArch64::ADDv16i8:
6030 case AArch64::ADDv4i16:
6031 case AArch64::ADDv8i16:
6032 case AArch64::ADDv2i32:
6033 case AArch64::ADDv4i32:
6034 case AArch64::ADDv1i64:
6035 case AArch64::ADDv2i64:
6036 case AArch64::MULv8i8:
6037 case AArch64::MULv16i8:
6038 case AArch64::MULv4i16:
6039 case AArch64::MULv8i16:
6040 case AArch64::MULv2i32:
6041 case AArch64::MULv4i32:
6042 case AArch64::ANDv8i8:
6043 case AArch64::ANDv16i8:
6044 case AArch64::ORRv8i8:
6045 case AArch64::ORRv16i8:
6046 case AArch64::EORv8i8:
6047 case AArch64::EORv16i8:
6049 case AArch64::ADD_ZZZ_B:
6050 case AArch64::ADD_ZZZ_H:
6051 case AArch64::ADD_ZZZ_S:
6052 case AArch64::ADD_ZZZ_D:
6053 case AArch64::MUL_ZZZ_B:
6054 case AArch64::MUL_ZZZ_H:
6055 case AArch64::MUL_ZZZ_S:
6056 case AArch64::MUL_ZZZ_D:
6057 case AArch64::AND_ZZZ:
6058 case AArch64::ORR_ZZZ:
6059 case AArch64::EOR_ZZZ:
6090 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6098 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6110 case AArch64::ADDWrr:
6112 "ADDWrr does not have register operands");
6113 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6114 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6116 case AArch64::ADDXrr:
6117 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6118 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6120 case AArch64::SUBWrr:
6121 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6122 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6124 case AArch64::SUBXrr:
6125 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6126 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6128 case AArch64::ADDWri:
6129 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6131 case AArch64::ADDXri:
6132 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6134 case AArch64::SUBWri:
6135 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6137 case AArch64::SUBXri:
6138 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6140 case AArch64::ADDv8i8:
6141 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6142 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6144 case AArch64::ADDv16i8:
6145 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6146 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6148 case AArch64::ADDv4i16:
6149 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6150 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6151 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6152 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6154 case AArch64::ADDv8i16:
6155 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6156 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6157 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6158 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6160 case AArch64::ADDv2i32:
6161 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6162 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6163 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6164 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6166 case AArch64::ADDv4i32:
6167 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6168 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6169 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6170 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6172 case AArch64::SUBv8i8:
6173 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6174 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6176 case AArch64::SUBv16i8:
6177 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6178 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6180 case AArch64::SUBv4i16:
6181 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6182 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
6183 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
6184 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
6186 case AArch64::SUBv8i16:
6187 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
6188 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
6189 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
6190 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
6192 case AArch64::SUBv2i32:
6193 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
6194 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
6195 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
6196 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
6198 case AArch64::SUBv4i32:
6199 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
6200 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
6201 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
6202 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
6219 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
6231 assert(
false &&
"Unsupported FP instruction in combiner\n");
6233 case AArch64::FADDHrr:
6235 "FADDHrr does not have register operands");
6237 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
6238 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
6240 case AArch64::FADDSrr:
6242 "FADDSrr does not have register operands");
6244 Found |=
Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
6245 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
6247 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
6248 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
6250 case AArch64::FADDDrr:
6251 Found |=
Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
6252 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
6254 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
6255 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
6257 case AArch64::FADDv4f16:
6258 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
6259 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
6261 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
6262 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
6264 case AArch64::FADDv8f16:
6265 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
6266 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
6268 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
6269 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
6271 case AArch64::FADDv2f32:
6272 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
6273 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
6275 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
6276 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
6278 case AArch64::FADDv2f64:
6279 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
6280 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
6282 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
6283 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
6285 case AArch64::FADDv4f32:
6286 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
6287 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
6289 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
6290 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
6292 case AArch64::FSUBHrr:
6293 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
6294 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
6295 Found |=
Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
6297 case AArch64::FSUBSrr:
6298 Found =
Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
6300 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
6301 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
6303 Found |=
Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
6305 case AArch64::FSUBDrr:
6306 Found =
Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
6308 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
6309 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
6311 Found |=
Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
6313 case AArch64::FSUBv4f16:
6314 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
6315 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
6317 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
6318 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
6320 case AArch64::FSUBv8f16:
6321 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
6322 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
6324 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
6325 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
6327 case AArch64::FSUBv2f32:
6328 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
6329 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
6331 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
6332 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
6334 case AArch64::FSUBv2f64:
6335 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
6336 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
6338 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
6339 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
6341 case AArch64::FSUBv4f32:
6342 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
6343 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
6345 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
6346 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
6357 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
6364 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
6365 MI->getOperand(1).getReg().isVirtual())
6366 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
6367 if (
MI &&
MI->getOpcode() == Opcode) {
6379 case AArch64::FMULv2f32:
6380 Found =
Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
6381 Found |=
Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
6383 case AArch64::FMULv2f64:
6384 Found =
Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
6385 Found |=
Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
6387 case AArch64::FMULv4f16:
6388 Found =
Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
6389 Found |=
Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
6391 case AArch64::FMULv4f32:
6392 Found =
Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
6393 Found |=
Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
6395 case AArch64::FMULv8f16:
6396 Found =
Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
6397 Found |=
Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
6410 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
6413 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
6414 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
6428 case AArch64::FNEGDr:
6430 case AArch64::FNEGSr:
6562 case AArch64::SUBWrr:
6563 case AArch64::SUBSWrr:
6564 case AArch64::SUBXrr:
6565 case AArch64::SUBSXrr:
6607 bool DoRegPressureReduce)
const {
6624 DoRegPressureReduce);
6653 const Register *ReplacedAddend =
nullptr) {
6654 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6656 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
6659 Register SrcReg0 = MUL->getOperand(1).getReg();
6660 bool Src0IsKill = MUL->getOperand(1).isKill();
6661 Register SrcReg1 = MUL->getOperand(2).getReg();
6662 bool Src1IsKill = MUL->getOperand(2).isKill();
6666 if (ReplacedAddend) {
6668 SrcReg2 = *ReplacedAddend;
6676 MRI.constrainRegClass(ResultReg, RC);
6678 MRI.constrainRegClass(SrcReg0, RC);
6680 MRI.constrainRegClass(SrcReg1, RC);
6682 MRI.constrainRegClass(SrcReg2, RC);
6685 if (kind == FMAInstKind::Default)
6690 else if (kind == FMAInstKind::Indexed)
6695 .
addImm(MUL->getOperand(3).getImm());
6696 else if (kind == FMAInstKind::Accumulator)
6702 assert(
false &&
"Invalid FMA instruction kind \n");
6716 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6717 Opc = AArch64::FNMADDSrrr;
6718 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
6719 Opc = AArch64::FNMADDDrrr;
6731 MRI.constrainRegClass(ResultReg, RC);
6733 MRI.constrainRegClass(SrcReg0, RC);
6735 MRI.constrainRegClass(SrcReg1, RC);
6737 MRI.constrainRegClass(SrcReg2, RC);
6753 unsigned IdxDupOp,
unsigned MulOpc,
6755 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
6756 "Invalid index of FMUL operand");
6764 if (Dup->
getOpcode() == TargetOpcode::COPY)
6768 MRI.clearKillFlags(DupSrcReg);
6769 MRI.constrainRegClass(DupSrcReg, RC);
6773 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
6797 FMAInstKind::Accumulator);
6814 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6829 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6831 FMAInstKind::Accumulator, &NewVR);
6843 FMAInstKind::Indexed);
6856 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6859 FMAInstKind::Indexed, &NewVR);
6884 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
6886 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6890 Register SrcReg0 = MUL->getOperand(1).getReg();
6891 bool Src0IsKill = MUL->getOperand(1).isKill();
6892 Register SrcReg1 = MUL->getOperand(2).getReg();
6893 bool Src1IsKill = MUL->getOperand(2).isKill();
6896 MRI.constrainRegClass(ResultReg, RC);
6898 MRI.constrainRegClass(SrcReg0, RC);
6900 MRI.constrainRegClass(SrcReg1, RC);
6902 MRI.constrainRegClass(VR, RC);
6924 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
6925 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
6935 Register NewVR =
MRI.createVirtualRegister(
MRI.getRegClass(RegA));
6938 if (Opcode == AArch64::SUBSWrr)
6939 Opcode = AArch64::SUBWrr;
6940 else if (Opcode == AArch64::SUBSXrr)
6941 Opcode = AArch64::SUBXrr;
6943 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
6944 "Unexpected instruction opcode.");
6961 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6988 DelInstrs, InstrIdxForVirtReg);
6994 InstrIdxForVirtReg);
7000 InstrIdxForVirtReg);
7009 Opc = AArch64::MADDWrrr;
7010 RC = &AArch64::GPR32RegClass;
7012 Opc = AArch64::MADDXrrr;
7013 RC = &AArch64::GPR64RegClass;
7024 Opc = AArch64::MADDWrrr;
7025 RC = &AArch64::GPR32RegClass;
7027 Opc = AArch64::MADDXrrr;
7028 RC = &AArch64::GPR64RegClass;
7040 unsigned BitSize, OrrOpc, ZeroReg;
7042 OrrOpc = AArch64::ORRWri;
7043 OrrRC = &AArch64::GPR32spRegClass;
7045 ZeroReg = AArch64::WZR;
7046 Opc = AArch64::MADDWrrr;
7047 RC = &AArch64::GPR32RegClass;
7049 OrrOpc = AArch64::ORRXri;
7050 OrrRC = &AArch64::GPR64spRegClass;
7052 ZeroReg = AArch64::XZR;
7053 Opc = AArch64::MADDXrrr;
7054 RC = &AArch64::GPR64RegClass;
7056 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7067 if (
Insn.size() != 1)
7069 auto MovI =
Insn.begin();
7072 if (MovI->Opcode == OrrOpc)
7078 assert((MovI->Opcode == AArch64::MOVNWi ||
7079 MovI->Opcode == AArch64::MOVZWi) &&
7082 assert((MovI->Opcode == AArch64::MOVNXi ||
7083 MovI->Opcode == AArch64::MOVZXi) &&
7090 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7091 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7102 unsigned SubOpc, ZeroReg;
7104 SubOpc = AArch64::SUBWrr;
7105 SubRC = &AArch64::GPR32spRegClass;
7106 ZeroReg = AArch64::WZR;
7107 Opc = AArch64::MADDWrrr;
7108 RC = &AArch64::GPR32RegClass;
7110 SubOpc = AArch64::SUBXrr;
7111 SubRC = &AArch64::GPR64spRegClass;
7112 ZeroReg = AArch64::XZR;
7113 Opc = AArch64::MADDXrrr;
7114 RC = &AArch64::GPR64RegClass;
7116 Register NewVR =
MRI.createVirtualRegister(SubRC);
7123 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7124 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7134 Opc = AArch64::MSUBWrrr;
7135 RC = &AArch64::GPR32RegClass;
7137 Opc = AArch64::MSUBXrrr;
7138 RC = &AArch64::GPR64RegClass;
7150 unsigned BitSize, OrrOpc, ZeroReg;
7152 OrrOpc = AArch64::ORRWri;
7153 OrrRC = &AArch64::GPR32spRegClass;
7155 ZeroReg = AArch64::WZR;
7156 Opc = AArch64::MADDWrrr;
7157 RC = &AArch64::GPR32RegClass;
7159 OrrOpc = AArch64::ORRXri;
7160 OrrRC = &AArch64::GPR64spRegClass;
7162 ZeroReg = AArch64::XZR;
7163 Opc = AArch64::MADDXrrr;
7164 RC = &AArch64::GPR64RegClass;
7166 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7176 if (
Insn.size() != 1)
7178 auto MovI =
Insn.begin();
7181 if (MovI->Opcode == OrrOpc)
7187 assert((MovI->Opcode == AArch64::MOVNWi ||
7188 MovI->Opcode == AArch64::MOVZWi) &&
7191 assert((MovI->Opcode == AArch64::MOVNXi ||
7192 MovI->Opcode == AArch64::MOVZXi) &&
7199 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7200 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7205 Opc = AArch64::MLAv8i8;
7206 RC = &AArch64::FPR64RegClass;
7210 Opc = AArch64::MLAv8i8;
7211 RC = &AArch64::FPR64RegClass;
7215 Opc = AArch64::MLAv16i8;
7216 RC = &AArch64::FPR128RegClass;
7220 Opc = AArch64::MLAv16i8;
7221 RC = &AArch64::FPR128RegClass;
7225 Opc = AArch64::MLAv4i16;
7226 RC = &AArch64::FPR64RegClass;
7230 Opc = AArch64::MLAv4i16;
7231 RC = &AArch64::FPR64RegClass;
7235 Opc = AArch64::MLAv8i16;
7236 RC = &AArch64::FPR128RegClass;
7240 Opc = AArch64::MLAv8i16;
7241 RC = &AArch64::FPR128RegClass;
7245 Opc = AArch64::MLAv2i32;
7246 RC = &AArch64::FPR64RegClass;
7250 Opc = AArch64::MLAv2i32;
7251 RC = &AArch64::FPR64RegClass;
7255 Opc = AArch64::MLAv4i32;
7256 RC = &AArch64::FPR128RegClass;
7260 Opc = AArch64::MLAv4i32;
7261 RC = &AArch64::FPR128RegClass;
7266 Opc = AArch64::MLAv8i8;
7267 RC = &AArch64::FPR64RegClass;
7269 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
7273 Opc = AArch64::MLSv8i8;
7274 RC = &AArch64::FPR64RegClass;
7278 Opc = AArch64::MLAv16i8;
7279 RC = &AArch64::FPR128RegClass;
7281 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
7285 Opc = AArch64::MLSv16i8;
7286 RC = &AArch64::FPR128RegClass;
7290 Opc = AArch64::MLAv4i16;
7291 RC = &AArch64::FPR64RegClass;
7293 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7297 Opc = AArch64::MLSv4i16;
7298 RC = &AArch64::FPR64RegClass;
7302 Opc = AArch64::MLAv8i16;
7303 RC = &AArch64::FPR128RegClass;
7305 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7309 Opc = AArch64::MLSv8i16;
7310 RC = &AArch64::FPR128RegClass;
7314 Opc = AArch64::MLAv2i32;
7315 RC = &AArch64::FPR64RegClass;
7317 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7321 Opc = AArch64::MLSv2i32;
7322 RC = &AArch64::FPR64RegClass;
7326 Opc = AArch64::MLAv4i32;
7327 RC = &AArch64::FPR128RegClass;
7329 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7333 Opc = AArch64::MLSv4i32;
7334 RC = &AArch64::FPR128RegClass;
7339 Opc = AArch64::MLAv4i16_indexed;
7340 RC = &AArch64::FPR64RegClass;
7344 Opc = AArch64::MLAv4i16_indexed;
7345 RC = &AArch64::FPR64RegClass;
7349 Opc = AArch64::MLAv8i16_indexed;
7350 RC = &AArch64::FPR128RegClass;
7354 Opc = AArch64::MLAv8i16_indexed;
7355 RC = &AArch64::FPR128RegClass;
7359 Opc = AArch64::MLAv2i32_indexed;
7360 RC = &AArch64::FPR64RegClass;
7364 Opc = AArch64::MLAv2i32_indexed;
7365 RC = &AArch64::FPR64RegClass;
7369 Opc = AArch64::MLAv4i32_indexed;
7370 RC = &AArch64::FPR128RegClass;
7374 Opc = AArch64::MLAv4i32_indexed;
7375 RC = &AArch64::FPR128RegClass;
7380 Opc = AArch64::MLAv4i16_indexed;
7381 RC = &AArch64::FPR64RegClass;
7383 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7387 Opc = AArch64::MLSv4i16_indexed;
7388 RC = &AArch64::FPR64RegClass;
7392 Opc = AArch64::MLAv8i16_indexed;
7393 RC = &AArch64::FPR128RegClass;
7395 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7399 Opc = AArch64::MLSv8i16_indexed;
7400 RC = &AArch64::FPR128RegClass;
7404 Opc = AArch64::MLAv2i32_indexed;
7405 RC = &AArch64::FPR64RegClass;
7407 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7411 Opc = AArch64::MLSv2i32_indexed;
7412 RC = &AArch64::FPR64RegClass;
7416 Opc = AArch64::MLAv4i32_indexed;
7417 RC = &AArch64::FPR128RegClass;
7419 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7423 Opc = AArch64::MLSv4i32_indexed;
7424 RC = &AArch64::FPR128RegClass;
7430 Opc = AArch64::FMADDHrrr;
7431 RC = &AArch64::FPR16RegClass;
7435 Opc = AArch64::FMADDSrrr;
7436 RC = &AArch64::FPR32RegClass;
7440 Opc = AArch64::FMADDDrrr;
7441 RC = &AArch64::FPR64RegClass;
7446 Opc = AArch64::FMADDHrrr;
7447 RC = &AArch64::FPR16RegClass;
7451 Opc = AArch64::FMADDSrrr;
7452 RC = &AArch64::FPR32RegClass;
7456 Opc = AArch64::FMADDDrrr;
7457 RC = &AArch64::FPR64RegClass;
7462 Opc = AArch64::FMLAv1i32_indexed;
7463 RC = &AArch64::FPR32RegClass;
7465 FMAInstKind::Indexed);
7468 Opc = AArch64::FMLAv1i32_indexed;
7469 RC = &AArch64::FPR32RegClass;
7471 FMAInstKind::Indexed);
7475 Opc = AArch64::FMLAv1i64_indexed;
7476 RC = &AArch64::FPR64RegClass;
7478 FMAInstKind::Indexed);
7481 Opc = AArch64::FMLAv1i64_indexed;
7482 RC = &AArch64::FPR64RegClass;
7484 FMAInstKind::Indexed);
7488 RC = &AArch64::FPR64RegClass;
7489 Opc = AArch64::FMLAv4i16_indexed;
7491 FMAInstKind::Indexed);
7494 RC = &AArch64::FPR64RegClass;
7495 Opc = AArch64::FMLAv4f16;
7497 FMAInstKind::Accumulator);
7500 RC = &AArch64::FPR64RegClass;
7501 Opc = AArch64::FMLAv4i16_indexed;
7503 FMAInstKind::Indexed);
7506 RC = &AArch64::FPR64RegClass;
7507 Opc = AArch64::FMLAv4f16;
7509 FMAInstKind::Accumulator);
7514 RC = &AArch64::FPR64RegClass;
7516 Opc = AArch64::FMLAv2i32_indexed;
7518 FMAInstKind::Indexed);
7520 Opc = AArch64::FMLAv2f32;
7522 FMAInstKind::Accumulator);
7527 RC = &AArch64::FPR64RegClass;
7529 Opc = AArch64::FMLAv2i32_indexed;
7531 FMAInstKind::Indexed);
7533 Opc = AArch64::FMLAv2f32;
7535 FMAInstKind::Accumulator);
7540 RC = &AArch64::FPR128RegClass;
7541 Opc = AArch64::FMLAv8i16_indexed;
7543 FMAInstKind::Indexed);
7546 RC = &AArch64::FPR128RegClass;
7547 Opc = AArch64::FMLAv8f16;
7549 FMAInstKind::Accumulator);
7552 RC = &AArch64::FPR128RegClass;
7553 Opc = AArch64::FMLAv8i16_indexed;
7555 FMAInstKind::Indexed);
7558 RC = &AArch64::FPR128RegClass;
7559 Opc = AArch64::FMLAv8f16;
7561 FMAInstKind::Accumulator);
7566 RC = &AArch64::FPR128RegClass;
7568 Opc = AArch64::FMLAv2i64_indexed;
7570 FMAInstKind::Indexed);
7572 Opc = AArch64::FMLAv2f64;
7574 FMAInstKind::Accumulator);
7579 RC = &AArch64::FPR128RegClass;
7581 Opc = AArch64::FMLAv2i64_indexed;
7583 FMAInstKind::Indexed);
7585 Opc = AArch64::FMLAv2f64;
7587 FMAInstKind::Accumulator);
7593 RC = &AArch64::FPR128RegClass;
7595 Opc = AArch64::FMLAv4i32_indexed;
7597 FMAInstKind::Indexed);
7599 Opc = AArch64::FMLAv4f32;
7601 FMAInstKind::Accumulator);
7607 RC = &AArch64::FPR128RegClass;
7609 Opc = AArch64::FMLAv4i32_indexed;
7611 FMAInstKind::Indexed);
7613 Opc = AArch64::FMLAv4f32;
7615 FMAInstKind::Accumulator);
7620 Opc = AArch64::FNMSUBHrrr;
7621 RC = &AArch64::FPR16RegClass;
7625 Opc = AArch64::FNMSUBSrrr;
7626 RC = &AArch64::FPR32RegClass;
7630 Opc = AArch64::FNMSUBDrrr;
7631 RC = &AArch64::FPR64RegClass;
7636 Opc = AArch64::FNMADDHrrr;
7637 RC = &AArch64::FPR16RegClass;
7641 Opc = AArch64::FNMADDSrrr;
7642 RC = &AArch64::FPR32RegClass;
7646 Opc = AArch64::FNMADDDrrr;
7647 RC = &AArch64::FPR64RegClass;
7652 Opc = AArch64::FMSUBHrrr;
7653 RC = &AArch64::FPR16RegClass;
7657 Opc = AArch64::FMSUBSrrr;
7658 RC = &AArch64::FPR32RegClass;
7662 Opc = AArch64::FMSUBDrrr;
7663 RC = &AArch64::FPR64RegClass;
7668 Opc = AArch64::FMLSv1i32_indexed;
7669 RC = &AArch64::FPR32RegClass;
7671 FMAInstKind::Indexed);
7675 Opc = AArch64::FMLSv1i64_indexed;
7676 RC = &AArch64::FPR64RegClass;
7678 FMAInstKind::Indexed);
7683 RC = &AArch64::FPR64RegClass;
7689 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7691 Opc = AArch64::FMLAv4f16;
7693 FMAInstKind::Accumulator, &NewVR);
7695 Opc = AArch64::FMLAv4i16_indexed;
7697 FMAInstKind::Indexed, &NewVR);
7702 RC = &AArch64::FPR64RegClass;
7703 Opc = AArch64::FMLSv4f16;
7705 FMAInstKind::Accumulator);
7708 RC = &AArch64::FPR64RegClass;
7709 Opc = AArch64::FMLSv4i16_indexed;
7711 FMAInstKind::Indexed);
7716 RC = &AArch64::FPR64RegClass;
7718 Opc = AArch64::FMLSv2i32_indexed;
7720 FMAInstKind::Indexed);
7722 Opc = AArch64::FMLSv2f32;
7724 FMAInstKind::Accumulator);
7730 RC = &AArch64::FPR128RegClass;
7736 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7738 Opc = AArch64::FMLAv8f16;
7740 FMAInstKind::Accumulator, &NewVR);
7742 Opc = AArch64::FMLAv8i16_indexed;
7744 FMAInstKind::Indexed, &NewVR);
7749 RC = &AArch64::FPR128RegClass;
7750 Opc = AArch64::FMLSv8f16;
7752 FMAInstKind::Accumulator);
7755 RC = &AArch64::FPR128RegClass;
7756 Opc = AArch64::FMLSv8i16_indexed;
7758 FMAInstKind::Indexed);
7763 RC = &AArch64::FPR128RegClass;
7765 Opc = AArch64::FMLSv2i64_indexed;
7767 FMAInstKind::Indexed);
7769 Opc = AArch64::FMLSv2f64;
7771 FMAInstKind::Accumulator);
7777 RC = &AArch64::FPR128RegClass;
7779 Opc = AArch64::FMLSv4i32_indexed;
7781 FMAInstKind::Indexed);
7783 Opc = AArch64::FMLSv4f32;
7785 FMAInstKind::Accumulator);
7790 RC = &AArch64::FPR64RegClass;
7796 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7798 Opc = AArch64::FMLAv2i32_indexed;
7800 FMAInstKind::Indexed, &NewVR);
7802 Opc = AArch64::FMLAv2f32;
7804 FMAInstKind::Accumulator, &NewVR);
7810 RC = &AArch64::FPR128RegClass;
7816 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7818 Opc = AArch64::FMLAv4i32_indexed;
7820 FMAInstKind::Indexed, &NewVR);
7822 Opc = AArch64::FMLAv4f32;
7824 FMAInstKind::Accumulator, &NewVR);
7830 RC = &AArch64::FPR128RegClass;
7836 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7838 Opc = AArch64::FMLAv2i64_indexed;
7840 FMAInstKind::Indexed, &NewVR);
7842 Opc = AArch64::FMLAv2f64;
7844 FMAInstKind::Accumulator, &NewVR);
7854 &AArch64::FPR128RegClass,
MRI);
7863 &AArch64::FPR128RegClass,
MRI);
7872 &AArch64::FPR128_loRegClass,
MRI);
7881 &AArch64::FPR128RegClass,
MRI);
7890 &AArch64::FPR128_loRegClass,
MRI);
7909 for (
auto *
MI : InsInstrs)
7910 MI->setFlags(Flags);
7951 bool IsNegativeBranch =
false;
7952 bool IsTestAndBranch =
false;
7953 unsigned TargetBBInMI = 0;
7954 switch (
MI.getOpcode()) {
7963 case AArch64::CBNZW:
7964 case AArch64::CBNZX:
7966 IsNegativeBranch =
true;
7971 IsTestAndBranch =
true;
7973 case AArch64::TBNZW:
7974 case AArch64::TBNZX:
7976 IsNegativeBranch =
true;
7977 IsTestAndBranch =
true;
7983 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
7987 assert(
MI.getParent() &&
"Incomplete machine instruciton\n");
8000 if (!
MRI->hasOneNonDBGUse(CopyVReg))
8002 if (!
MRI->hasOneDef(CopyVReg))
8011 case AArch64::ANDWri:
8012 case AArch64::ANDXri: {
8013 if (IsTestAndBranch)
8017 if (!
MRI->hasOneNonDBGUse(VReg))
8031 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
8037 unsigned Opc = (Imm < 32)
8038 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
8039 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
8052 if (!Is32Bit && Imm < 32)
8054 MI.eraseFromParent();
8058 case AArch64::CSINCWr:
8059 case AArch64::CSINCXr: {
8079 if (IsNegativeBranch)
8082 MI.eraseFromParent();
8088std::pair<unsigned, unsigned>
8091 return std::make_pair(TF & Mask, TF & ~Mask);
8096 using namespace AArch64II;
8098 static const std::pair<unsigned, const char *> TargetFlags[] = {
8099 {MO_PAGE,
"aarch64-page"}, {MO_PAGEOFF,
"aarch64-pageoff"},
8100 {MO_G3,
"aarch64-g3"}, {MO_G2,
"aarch64-g2"},
8101 {MO_G1,
"aarch64-g1"}, {MO_G0,
"aarch64-g0"},
8102 {MO_HI12,
"aarch64-hi12"}};
8108 using namespace AArch64II;
8110 static const std::pair<unsigned, const char *> TargetFlags[] = {
8111 {MO_COFFSTUB,
"aarch64-coffstub"},
8112 {MO_GOT,
"aarch64-got"},
8113 {MO_NC,
"aarch64-nc"},
8114 {MO_S,
"aarch64-s"},
8115 {MO_TLS,
"aarch64-tls"},
8116 {MO_DLLIMPORT,
"aarch64-dllimport"},
8117 {MO_PREL,
"aarch64-prel"},
8118 {MO_TAGGED,
"aarch64-tagged"},
8119 {MO_ARM64EC_CALLMANGLE,
"aarch64-arm64ec-callmangle"},
8126 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8234 for (
unsigned Reg : AArch64::GPR64RegClass) {
8236 Reg != AArch64::LR &&
8237 Reg != AArch64::X16 &&
8238 Reg != AArch64::X17 &&
8239 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
8240 C.isAvailableInsideSeq(
Reg,
TRI))
8271 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
8274std::optional<outliner::OutlinedFunction>
8276 std::vector<outliner::Candidate> &RepeatedSequenceLocs)
const {
8277 unsigned SequenceSize = 0;
8278 for (
auto &
MI : RepeatedSequenceLocs[0])
8281 unsigned NumBytesToCreateFrame = 0;
8291 if (std::adjacent_find(
8292 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
8296 if (outliningCandidatesSigningScopeConsensus(a, b) &&
8297 outliningCandidatesSigningKeyConsensus(a, b) &&
8298 outliningCandidatesV8_3OpsConsensus(a, b)) {
8302 }) != RepeatedSequenceLocs.end()) {
8303 return std::nullopt;
8320 unsigned NumBytesToCheckLRInTCEpilogue = 0;
8321 if (RepeatedSequenceLocs[0]
8323 ->getInfo<AArch64FunctionInfo>()
8324 ->shouldSignReturnAddress(
true)) {
8326 NumBytesToCreateFrame += 8;
8329 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod();
8330 NumBytesToCheckLRInTCEpilogue =
8334 if (isTailCallReturnInst(RepeatedSequenceLocs[0].back()))
8335 SequenceSize += NumBytesToCheckLRInTCEpilogue;
8343 for (
auto &
MI :
C) {
8344 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
8345 switch (
MI.getOpcode()) {
8346 case AArch64::ADDXri:
8347 case AArch64::ADDWri:
8348 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8350 "Expected operand to be immediate");
8352 "Expected operand to be a register");
8356 if (
MI.getOperand(1).getReg() == AArch64::SP)
8357 SPValue +=
MI.getOperand(2).getImm();
8361 case AArch64::SUBXri:
8362 case AArch64::SUBWri:
8363 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8365 "Expected operand to be immediate");
8367 "Expected operand to be a register");
8371 if (
MI.getOperand(1).getReg() == AArch64::SP)
8372 SPValue -=
MI.getOperand(2).getImm();
8389 if (RepeatedSequenceLocs.size() < 2)
8390 return std::nullopt;
8394 unsigned FlagsSetInAll = 0xF;
8398 FlagsSetInAll &=
C.Flags;
8400 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
8403 auto SetCandidateCallInfo =
8404 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
8406 C.setCallInfo(CallID, NumBytesForCall);
8410 NumBytesToCreateFrame += 4;
8413 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
8418 unsigned CFICount = 0;
8419 for (
auto &
I : RepeatedSequenceLocs[0]) {
8420 if (
I.isCFIInstruction())
8430 std::vector<MCCFIInstruction> CFIInstructions =
8431 C.getMF()->getFrameInstructions();
8433 if (CFICount > 0 && CFICount != CFIInstructions.size())
8434 return std::nullopt;
8442 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
8443 !
MI.readsRegister(AArch64::SP, &
TRI))
8449 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
8454 if (
MI.mayLoadOrStore()) {
8457 bool OffsetIsScalable;
8461 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
8462 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
8466 if (OffsetIsScalable)
8474 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
8475 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
8478 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
8479 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
8494 bool AllStackInstrsSafe =
8499 if (RepeatedSequenceLocs[0].back().isTerminator()) {
8501 NumBytesToCreateFrame = 0;
8502 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
8506 else if (LastInstrOpcode == AArch64::BL ||
8507 ((LastInstrOpcode == AArch64::BLR ||
8508 LastInstrOpcode == AArch64::BLRNoIP) &&
8512 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
8520 unsigned NumBytesNoStackCalls = 0;
8521 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
8526 (
C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
8527 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
8536 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
8539 if (LRAvailable && !IsNoReturn) {
8540 NumBytesNoStackCalls += 4;
8542 CandidatesWithoutStackFixups.push_back(
C);
8547 else if (findRegisterToSaveLRTo(
C)) {
8548 NumBytesNoStackCalls += 12;
8550 CandidatesWithoutStackFixups.push_back(
C);
8555 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
8556 NumBytesNoStackCalls += 12;
8558 CandidatesWithoutStackFixups.push_back(
C);
8564 NumBytesNoStackCalls += SequenceSize;
8571 if (!AllStackInstrsSafe ||
8572 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
8573 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
8575 if (RepeatedSequenceLocs.size() < 2)
8576 return std::nullopt;
8625 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8629 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
8630 !findRegisterToSaveLRTo(
C));
8636 if (RepeatedSequenceLocs.size() < 2) {
8637 RepeatedSequenceLocs.clear();
8638 return std::nullopt;
8644 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8648 bool ModStackToSaveLR =
false;
8649 if (std::any_of(FirstCand.
begin(), std::prev(FirstCand.
end()),
8651 ModStackToSaveLR =
true;
8660 ModStackToSaveLR =
true;
8662 if (ModStackToSaveLR) {
8664 if (!AllStackInstrsSafe) {
8665 RepeatedSequenceLocs.clear();
8666 return std::nullopt;
8670 NumBytesToCreateFrame += 8;
8677 return std::nullopt;
8680 NumBytesToCreateFrame, FrameID);
8684 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
8688 const auto &CFn = Candidates.front().getMF()->getFunction();
8692 if (CFn.hasFnAttribute(
"sign-return-address"))
8693 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
8694 if (CFn.hasFnAttribute(
"sign-return-address-key"))
8695 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
8697 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
8705 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
8719 if (!AFI || AFI->
hasRedZone().value_or(
true))
8739 unsigned &Flags)
const {
8741 "Must track liveness!");
8743 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
8758 auto AreAllUnsafeRegsDead = [&LRU]() {
8775 bool LRAvailableEverywhere =
true;
8780 if (
MI.isCall() && !
MI.isTerminator())
8781 Flags |= MachineOutlinerMBBFlags::HasCalls;
8786 auto CreateNewRangeStartingAt =
8787 [&RangeBegin, &RangeEnd,
8789 RangeBegin = NewBegin;
8790 RangeEnd = std::next(RangeBegin);
8793 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
8798 Ranges.push_back(std::make_pair(RangeBegin, RangeEnd));
8806 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
8810 UpdateWholeMBBFlags(*FirstPossibleEndPt);
8811 if (AreAllUnsafeRegsDead())
8818 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
8824 UpdateWholeMBBFlags(
MI);
8825 if (!AreAllUnsafeRegsDead()) {
8826 SaveRangeIfNonEmpty();
8827 CreateNewRangeStartingAt(
MI.getIterator());
8830 LRAvailableEverywhere &= LRU.
available(AArch64::LR);
8831 RangeBegin =
MI.getIterator();
8836 if (AreAllUnsafeRegsDead())
8837 SaveRangeIfNonEmpty();
8842 std::reverse(Ranges.begin(), Ranges.end());
8845 if (!LRAvailableEverywhere)
8846 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
8852 unsigned Flags)
const {
8860 switch (
MI.getOpcode()) {
8862 case AArch64::PACIASP:
8863 case AArch64::PACIBSP:
8864 case AArch64::PACIASPPC:
8865 case AArch64::PACIBSPPC:
8866 case AArch64::AUTIASP:
8867 case AArch64::AUTIBSP:
8868 case AArch64::AUTIASPPCi:
8869 case AArch64::AUTIASPPCr:
8870 case AArch64::AUTIBSPPCi:
8871 case AArch64::AUTIBSPPCr:
8872 case AArch64::RETAA:
8873 case AArch64::RETAB:
8874 case AArch64::RETAASPPCi:
8875 case AArch64::RETAASPPCr:
8876 case AArch64::RETABSPPCi:
8877 case AArch64::RETABSPPCr:
8878 case AArch64::EMITBKEY:
8879 case AArch64::PAUTH_PROLOGUE:
8880 case AArch64::PAUTH_EPILOGUE:
8894 if (
MI.isCFIInstruction())
8898 if (
MI.isTerminator())
8907 assert(!MOP.isCFIIndex());
8910 if (MOP.isReg() && !MOP.isImplicit() &&
8911 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
8918 if (
MI.getOpcode() == AArch64::ADRP)
8939 if (MOP.isGlobal()) {
8940 Callee = dyn_cast<Function>(MOP.getGlobal());
8947 if (Callee && Callee->getName() ==
"\01_mcount")
8955 if (
MI.getOpcode() == AArch64::BLR ||
8956 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
8960 return UnknownCallOutlineType;
8968 return UnknownCallOutlineType;
8976 return UnknownCallOutlineType;
9001 bool OffsetIsScalable;
9004 if (!
MI.mayLoadOrStore() ||
9007 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
9012 int64_t Dummy1, Dummy2;
9015 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
9017 assert(Scale != 0 &&
"Unexpected opcode!");
9018 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
9023 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
9024 StackOffsetOperand.
setImm(NewImm);
9030 bool ShouldSignReturnAddr) {
9031 if (!ShouldSignReturnAddr)
9037 TII->get(AArch64::PAUTH_EPILOGUE))
9053 unsigned TailOpcode;
9054 if (Call->getOpcode() == AArch64::BL) {
9055 TailOpcode = AArch64::TCRETURNdi;
9057 assert(Call->getOpcode() == AArch64::BLR ||
9058 Call->getOpcode() == AArch64::BLRNoIP);
9059 TailOpcode = AArch64::TCRETURNriALL;
9062 .
add(Call->getOperand(0))
9065 Call->eraseFromParent();
9070 bool IsLeafFunction =
true;
9074 return MI.isCall() && !
MI.isReturn();
9084 "Can only fix up stack references once");
9085 fixupPostOutline(
MBB);
9087 IsLeafFunction =
false;
9098 Et = std::prev(
MBB.
end());
9111 unsigned DwarfReg =
MRI->getDwarfRegNum(AArch64::LR,
true);
9114 int64_t StackPosEntry =
9167 fixupPostOutline(
MBB);
9178 .addGlobalAddress(M.getNamedValue(MF.
getName()))
9188 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9203 Register Reg = findRegisterToSaveLRTo(
C);
9204 assert(Reg &&
"No callee-saved register available?");
9238 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9254 bool AllowSideEffects)
const {
9259 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
9261 }
else if (STI.hasSVE()) {
9271std::optional<DestSourcePair>
9276 if (
MI.getOpcode() == AArch64::ORRWrs &&
9277 MI.getOperand(1).getReg() == AArch64::WZR &&
9278 MI.getOperand(3).getImm() == 0x0 &&
9280 (!
MI.getOperand(0).getReg().isVirtual() ||
9281 MI.getOperand(0).getSubReg() == 0) &&
9282 (!
MI.getOperand(0).getReg().isPhysical() ||
9283 MI.findRegisterDefOperandIdx(
MI.getOperand(0).getReg() - AArch64::W0 +
9288 if (
MI.getOpcode() == AArch64::ORRXrs &&
9289 MI.getOperand(1).getReg() == AArch64::XZR &&
9290 MI.getOperand(3).getImm() == 0x0)
9293 return std::nullopt;
9296std::optional<DestSourcePair>
9298 if (
MI.getOpcode() == AArch64::ORRWrs &&
9299 MI.getOperand(1).getReg() == AArch64::WZR &&
9300 MI.getOperand(3).getImm() == 0x0)
9302 return std::nullopt;
9305std::optional<RegImmPair>
9314 return std::nullopt;
9316 switch (
MI.getOpcode()) {
9318 return std::nullopt;
9319 case AArch64::SUBWri:
9320 case AArch64::SUBXri:
9321 case AArch64::SUBSWri:
9322 case AArch64::SUBSXri:
9325 case AArch64::ADDSWri:
9326 case AArch64::ADDSXri:
9327 case AArch64::ADDWri:
9328 case AArch64::ADDXri: {
9330 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
9331 !
MI.getOperand(2).isImm())
9332 return std::nullopt;
9333 int Shift =
MI.getOperand(3).getImm();
9334 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
9335 Offset = Sign * (
MI.getOperand(2).getImm() << Shift);
9344static std::optional<ParamLoadedValue>
9348 auto DestSrc =
TII->isCopyLikeInstr(
MI);
9350 return std::nullopt;
9352 Register DestReg = DestSrc->Destination->getReg();
9353 Register SrcReg = DestSrc->Source->getReg();
9358 if (DestReg == DescribedReg)
9362 if (
MI.getOpcode() == AArch64::ORRWrs &&
9363 TRI->isSuperRegister(DestReg, DescribedReg))
9367 if (
MI.getOpcode() == AArch64::ORRXrs &&
9368 TRI->isSubRegister(DestReg, DescribedReg)) {
9369 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
9373 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
9374 "Unhandled ORR[XW]rs copy case");
9376 return std::nullopt;
9395 return MI.getOpcode() == AArch64::INLINEASM_BR;
9413 switch (
MI.getOpcode()) {
9414 case TargetOpcode::G_BRJT:
9415 case AArch64::JumpTableDest32:
9416 case AArch64::JumpTableDest16:
9417 case AArch64::JumpTableDest8:
9428std::optional<ParamLoadedValue>
9433 switch (
MI.getOpcode()) {
9434 case AArch64::MOVZWi:
9435 case AArch64::MOVZXi: {
9438 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
9439 return std::nullopt;
9441 if (!
MI.getOperand(1).isImm())
9442 return std::nullopt;
9443 int64_t Immediate =
MI.getOperand(1).getImm();
9444 int Shift =
MI.getOperand(2).getImm();
9448 case AArch64::ORRWrs:
9449 case AArch64::ORRXrs:
9459 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
9460 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
9463 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
9467 if (!
MRI.hasOneNonDBGUse(DefReg))
9472 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
9473 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
9494 unsigned Scale)
const {
9505 unsigned Shift =
Log2_64(NumBytes);
9506 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
9514 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
9519 return AArch64::BLRNoIP;
9521 return AArch64::BLR;
9526 Register TargetReg,
bool FrameSetup)
const {
9527 assert(TargetReg != AArch64::SP &&
"New top of stack cannot aleady be in SP");
9539 MF.
insert(MBBInsertPoint, LoopTestMBB);
9542 MF.
insert(MBBInsertPoint, LoopBodyMBB);
9544 MF.
insert(MBBInsertPoint, ExitMBB);
9554 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
9568 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
9581 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
9606 return ExitMBB->
begin();
9623 unsigned CompCounterOprNum;
9627 unsigned UpdateCounterOprNum;
9631 bool IsUpdatePriorComp;
9643 TII(MF->getSubtarget().getInstrInfo()),
9644 TRI(MF->getSubtarget().getRegisterInfo()),
MRI(MF->getRegInfo()),
9645 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
9646 CompCounterOprNum(CompCounterOprNum), Update(Update),
9647 UpdateCounterOprNum(UpdateCounterOprNum),
Init(
Init),
9648 IsUpdatePriorComp(IsUpdatePriorComp),
Cond(
Cond.begin(),
Cond.end()) {}
9650 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
9656 std::optional<bool> createTripCountGreaterCondition(
9666 void createRemainingIterationsGreaterCondition(
9672 void adjustTripCount(
int TripCountAdjust)
override {}
9674 void disposed()
override {}
9675 bool isMVEExpanderSupported()
override {
return true; }
9693 Result =
MRI.createVirtualRegister(
9696 }
else if (
I == ReplaceOprNum) {
9697 MRI.constrainRegClass(
9707void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
9723 assert(CondBranch->getOpcode() == AArch64::Bcc);
9727 if (CondBranch->getOperand(1).getMBB() == LoopBB)
9734 auto AccumulateCond = [&](
Register CurCond,
9736 Register NewCond =
MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
9745 if (!LastStage0Insts.
empty() && LastStage0Insts[Comp]->getParent() == &
MBB) {
9749 for (
int I = 0;
I <= TC; ++
I) {
9755 AccCond = AccumulateCond(AccCond,
CC);
9759 if (Update != Comp && IsUpdatePriorComp) {
9761 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
9762 NextCounter =
cloneInstr(Update, UpdateCounterOprNum, Counter,
MBB,
9766 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg();
9768 }
else if (Update != Comp) {
9773 Counter = NextCounter;
9777 if (LastStage0Insts.
empty()) {
9781 if (IsUpdatePriorComp)
9786 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg();
9789 for (
int I = 0;
I <= TC; ++
I) {
9793 AccCond = AccumulateCond(AccCond,
CC);
9794 if (
I != TC && Update != Comp)
9797 Counter = NextCounter;
9813 assert(Phi.getNumOperands() == 5);
9814 if (Phi.getOperand(2).getMBB() ==
MBB) {
9815 RegMBB = Phi.getOperand(1).getReg();
9816 RegOther = Phi.getOperand(3).getReg();
9818 assert(Phi.getOperand(4).getMBB() ==
MBB);
9819 RegMBB = Phi.getOperand(3).getReg();
9820 RegOther = Phi.getOperand(1).getReg();
9825 if (!Reg.isVirtual())
9828 return MRI.getVRegDef(Reg)->getParent() != BB;
9834 unsigned &UpdateCounterOprNum,
Register &InitReg,
9835 bool &IsUpdatePriorComp) {
9849 if (!Reg.isVirtual())
9852 UpdateInst =
nullptr;
9853 UpdateCounterOprNum = 0;
9855 IsUpdatePriorComp =
true;
9859 if (Def->getParent() != LoopBB)
9861 if (Def->isCopy()) {
9863 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg())
9865 CurReg = Def->getOperand(1).getReg();
9866 }
else if (Def->isPHI()) {
9870 IsUpdatePriorComp =
false;
9875 switch (Def->getOpcode()) {
9876 case AArch64::ADDSXri:
9877 case AArch64::ADDSWri:
9878 case AArch64::SUBSXri:
9879 case AArch64::SUBSWri:
9880 case AArch64::ADDXri:
9881 case AArch64::ADDWri:
9882 case AArch64::SUBXri:
9883 case AArch64::SUBWri:
9885 UpdateCounterOprNum = 1;
9887 case AArch64::ADDSXrr:
9888 case AArch64::ADDSWrr:
9889 case AArch64::SUBSXrr:
9890 case AArch64::SUBSWrr:
9891 case AArch64::ADDXrr:
9892 case AArch64::ADDWrr:
9893 case AArch64::SUBXrr:
9894 case AArch64::SUBWrr:
9897 UpdateCounterOprNum = 1;
9899 UpdateCounterOprNum = 2;
9906 CurReg = Def->getOperand(UpdateCounterOprNum).getReg();
9921std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
9932 if (
MI.isCall() ||
MI.hasUnmodeledSideEffects())
9943 if (
TBB == LoopBB && FBB == LoopBB)
9947 if (
TBB != LoopBB && FBB ==
nullptr)
9950 assert((
TBB == LoopBB || FBB == LoopBB) &&
9951 "The Loop must be a single-basic-block loop");
9956 if (CondBranch->
getOpcode() != AArch64::Bcc)
9964 unsigned CompCounterOprNum = 0;
9966 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
9970 switch (
MI.getOpcode()) {
9971 case AArch64::SUBSXri:
9972 case AArch64::SUBSWri:
9973 case AArch64::ADDSXri:
9974 case AArch64::ADDSWri:
9976 CompCounterOprNum = 1;
9978 case AArch64::ADDSWrr:
9979 case AArch64::ADDSXrr:
9980 case AArch64::SUBSWrr:
9981 case AArch64::SUBSXrr:
9992 if (CompCounterOprNum == 0) {
9994 CompCounterOprNum = 2;
9996 CompCounterOprNum = 1;
10008 bool IsUpdatePriorComp;
10009 unsigned UpdateCounterOprNum;
10011 Update, UpdateCounterOprNum,
Init, IsUpdatePriorComp))
10014 return std::make_unique<AArch64PipelinerLoopInfo>(
10015 LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,
10019#define GET_INSTRINFO_HELPERS
10020#define GET_INSTRMAP_INFO
10021#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
If Reg is an induction variable, return true and set some parameters.
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static unsigned getBranchDisplacementBits(unsigned Opc)
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
Clone an instruction from MI.
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, unsigned Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const Function * getParent(const Value *V)
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
const SetOfInstructions & getLOHRelated() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
void setOutliningStyle(std::string Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors, int64_t &NumDataVectors)
Returns the offset in parts to which this frame offset can be decomposed for the purpose of describin...
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
uint64_t getElementSizeForOpcode(unsigned Opc) const
Returns the vector element size (B, H, S or D) of an SVE opcode.
outliner::InstrType getOutliningTypeImpl(MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
bool isWhileOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE WHILE## instruction.
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
SmallVector< std::pair< MachineBasicBlock::iterator, MachineBasicBlock::iterator > > getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
bool useMachineCombiner() const override
AArch64 supports MachineCombiner.
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const override
static bool isFalkorShiftExtFast(const MachineInstr &MI)
Returns true if the instruction has a shift by immediate that can be executed in one cycle less.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
bool expandPostRAPseudo(MachineInstr &MI) const override
unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
static bool isFpOrNEON(const MachineInstr &MI)
Returns whether the instruction is FP or NEON.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL, unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
bool isFunctionSafeToSplit(const MachineFunction &MF) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
Return true when Inst is associative and commutative so that it can be reassociated.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
std::optional< outliner::OutlinedFunction > getOutliningCandidateInfo(std::vector< outliner::Candidate > &RepeatedSequenceLocs) const override
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset, unsigned Scale) const
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
ArrayRef< std::pair< unsigned, const char * > > getSerializableBitmaskMachineOperandTargetFlags() const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isPTestLikeOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE instruction that sets the condition codes as if it's results...
void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const override
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
bool isTargetILP32() const
bool isSVEorStreamingSVEAvailable() const
Returns true if the target has access to either the full range of SVE instructions,...
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MBBSectionID getSectionID() const
Returns the section ID of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumOperands() const
Retuns the total number of operands.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool registerDefIsDead(Register Reg, const TargetRegisterInfo *TRI) const
Returns true if the register is dead in this machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
void addRegisterDefined(Register Reg, const TargetRegisterInfo *RegInfo=nullptr)
We have determined MI defines a register.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
Primary interface to the complete machine description for the target machine.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
static bool isIndirectBranchOpcode(int Opc)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
unsigned getUndefRegState(bool B)
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
DWARFExpression::Operation Op
static bool isUncondBranchOpcode(int Opc)
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
static const MachineMemOperand::Flags MOStridedAccess
@ Default
The result values are uniform if and only if all operands are uniform.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
static const MBBSectionID ColdSectionID
MachineJumpTableEntry - One jump table in the jump table info.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
MachineBasicBlock::iterator begin()
MachineBasicBlock::iterator end()
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.