22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
49 AArch64DAGToDAGISel() =
delete;
65 unsigned ConstraintID,
66 std::vector<SDValue> &OutOps)
override;
68 template <
signed Low,
signed High,
signed Scale>
71 bool tryMLAV64LaneV128(
SDNode *
N);
72 bool tryMULLV64LaneV128(
unsigned IntNo,
SDNode *
N);
78 return SelectShiftedRegister(
N,
false, Reg, Shift);
81 return SelectShiftedRegister(
N,
true, Reg, Shift);
84 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
87 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
90 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
93 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
96 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
99 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
102 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
105 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
108 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
111 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
114 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
117 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
120 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
123 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
126 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
129 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
132 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
134 template <
unsigned Size,
unsigned Max>
138 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
140 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
141 int64_t
C = CI->getSExtValue();
156 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
162 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
167 N =
N->getOperand(0);
169 !isa<ConstantSDNode>(
N->getOperand(1)))
171 EVT VT =
N->getValueType(0);
172 EVT LVT =
N->getOperand(0).getValueType();
173 unsigned Index =
N->getConstantOperandVal(1);
177 Res =
N->getOperand(0);
185 EVT VT = Op.getValueType();
186 unsigned ShtAmt =
N->getConstantOperandVal(1);
193 Op.getOperand(1).getConstantOperandVal(0)
194 << Op.getOperand(1).getConstantOperandVal(1));
196 isa<ConstantSDNode>(Op.getOperand(1).getOperand(0)))
198 Op.getOperand(1).getConstantOperandVal(0));
202 if (Imm != 1ULL << (ShtAmt - 1))
205 Res1 = Op.getOperand(0);
210 bool SelectDupZeroOrUndef(
SDValue N) {
211 switch(
N->getOpcode()) {
216 auto Opnd0 =
N->getOperand(0);
231 switch(
N->getOpcode()) {
234 auto Opnd0 =
N->getOperand(0);
246 template<MVT::SimpleValueType VT>
248 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
251 template <MVT::SimpleValueType VT>
253 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
256 template <MVT::SimpleValueType VT,
bool Invert = false>
258 return SelectSVELogicalImm(
N, VT, Imm, Invert);
261 template <MVT::SimpleValueType VT>
263 return SelectSVEArithImm(
N, VT, Imm);
266 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
268 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
275 EVT EltVT =
N->getValueType(0).getVectorElementType();
276 return SelectSVEShiftImm(
N->getOperand(0), 1,
282 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
284 if (!isa<ConstantSDNode>(
N))
287 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
289 MulImm = 1LL << MulImm;
291 if ((MulImm % std::abs(Scale)) != 0)
295 if ((MulImm >= Min) && (MulImm <= Max)) {
303 template <
signed Max,
signed Scale>
305 if (!isa<ConstantSDNode>(
N))
308 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
310 if (MulImm >= 0 && MulImm <= Max) {
320 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
322 Imm = CurDAG->getRegister(BaseReg +
C,
MVT::Other);
345 const unsigned SubRegs[]);
347 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
349 bool tryIndexedLoad(
SDNode *
N);
351 bool trySelectStackSlotTagP(
SDNode *
N);
354 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
356 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
358 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
359 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
360 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
361 unsigned Opc_rr,
unsigned Opc_ri,
362 bool IsIntr =
false);
363 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
364 bool IsZmMulti,
unsigned Opcode);
365 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
366 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
367 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
368 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
369 bool IsTupleInput,
unsigned Opc);
370 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
372 template <
unsigned MaxIdx,
unsigned Scale>
373 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
378 template <
int64_t Min,
int64_t Max>
382 template <
unsigned Scale>
384 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
387 template <
unsigned MaxIdx,
unsigned Scale>
392 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
393 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
394 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
395 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
396 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
397 unsigned Opc_rr,
unsigned Opc_ri);
398 std::tuple<unsigned, SDValue, SDValue>
399 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
403 bool tryBitfieldExtractOp(
SDNode *
N);
404 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
405 bool tryBitfieldInsertOp(
SDNode *
N);
406 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
407 bool tryShiftAmountMod(
SDNode *
N);
410 bool tryReadRegister(
SDNode *
N);
411 bool tryWriteRegister(
SDNode *
N);
413 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
414 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
417#include "AArch64GenDAGISel.inc"
425 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
427 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
440 bool isWorthFolding(
SDValue V)
const;
441 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
444 template<
unsigned RegW
idth>
446 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
449 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
451 bool SelectCMP_SWAP(
SDNode *
N);
459 bool AllowSaturation,
SDValue &Imm);
467 bool SelectAllActivePredicate(
SDValue N);
472char AArch64DAGToDAGISel::ID = 0;
480 Imm =
C->getZExtValue();
497 return N->getOpcode() == Opc &&
508 return Imm == ImmExpected;
512bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
513 const SDValue &Op,
unsigned ConstraintID, std::vector<SDValue> &OutOps) {
514 switch(ConstraintID) {
527 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
528 dl,
Op.getValueType(),
530 OutOps.push_back(NewOp);
546 if (!isa<ConstantSDNode>(
N.getNode()))
549 uint64_t Immed = cast<ConstantSDNode>(
N.getNode())->getZExtValue();
552 if (Immed >> 12 == 0) {
554 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
562 Val = CurDAG->getTargetConstant(Immed, dl,
MVT::i32);
563 Shift = CurDAG->getTargetConstant(ShVal, dl,
MVT::i32);
576 if (!isa<ConstantSDNode>(
N.getNode()))
580 uint64_t Immed = cast<ConstantSDNode>(
N.getNode())->getZExtValue();
591 Immed = ~Immed + 1ULL;
592 if (Immed & 0xFFFFFFFFFF000000ULL)
595 Immed &= 0xFFFFFFULL;
596 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N),
MVT::i32), Val,
603 switch (
N.getOpcode()) {
622 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
625 unsigned ShiftVal = CSD->getZExtValue();
634 if (!isa<MemSDNode>(*UI))
636 if (!isa<MemSDNode>(*UII))
642bool AArch64DAGToDAGISel::isWorthFolding(
SDValue V)
const {
645 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
649 if (Subtarget->hasLSLFast() &&
V.getOpcode() ==
ISD::SHL &&
652 if (Subtarget->hasLSLFast() &&
V.getOpcode() ==
ISD::ADD) {
667bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
669 EVT VT =
N.getValueType();
673 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
679 unsigned LHSOpcode =
LHS->getOpcode();
693 unsigned LowZBits, MaskLen;
697 unsigned BitWidth =
N.getValueSizeInBits();
704 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
707 NewShiftC = LowZBits - ShiftAmtC;
708 NewShiftOp = VT ==
MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
714 NewShiftC = LowZBits + ShiftAmtC;
727 NewShiftOp = VT ==
MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
729 NewShiftOp = VT ==
MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
733 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
735 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
736 NewShiftAmt, BitWidthMinus1),
739 Shift = CurDAG->getTargetConstant(ShVal,
DL,
MVT::i32);
748bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
750 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
760 unsigned BitSize =
N.getValueSizeInBits();
761 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
764 Reg =
N.getOperand(0);
766 return isWorthFolding(
N);
780 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
782 SrcVT =
N.getOperand(0).getValueType();
784 if (!IsLoadStore && SrcVT ==
MVT::i8)
786 else if (!IsLoadStore && SrcVT ==
MVT::i16)
795 EVT SrcVT =
N.getOperand(0).getValueType();
796 if (!IsLoadStore && SrcVT ==
MVT::i8)
798 else if (!IsLoadStore && SrcVT ==
MVT::i16)
851 SDValue &LaneOp,
int &LaneIdx) {
865bool AArch64DAGToDAGISel::tryMLAV64LaneV128(
SDNode *
N) {
885 SDValue Ops[] = { Op0, MLAOp1, MLAOp2, LaneIdxVal };
887 unsigned MLAOpc = ~0
U;
889 switch (
N->getSimpleValueType(0).SimpleTy) {
893 MLAOpc = AArch64::MLAv4i16_indexed;
896 MLAOpc = AArch64::MLAv8i16_indexed;
899 MLAOpc = AArch64::MLAv2i32_indexed;
902 MLAOpc = AArch64::MLAv4i32_indexed;
906 ReplaceNode(
N, CurDAG->getMachineNode(MLAOpc, dl,
N->getValueType(0), Ops));
910bool AArch64DAGToDAGISel::tryMULLV64LaneV128(
unsigned IntNo,
SDNode *
N) {
922 SDValue Ops[] = { SMULLOp0, SMULLOp1, LaneIdxVal };
924 unsigned SMULLOpc = ~0
U;
926 if (IntNo == Intrinsic::aarch64_neon_smull) {
927 switch (
N->getSimpleValueType(0).SimpleTy) {
931 SMULLOpc = AArch64::SMULLv4i16_indexed;
934 SMULLOpc = AArch64::SMULLv2i32_indexed;
937 }
else if (IntNo == Intrinsic::aarch64_neon_umull) {
938 switch (
N->getSimpleValueType(0).SimpleTy) {
942 SMULLOpc = AArch64::UMULLv4i16_indexed;
945 SMULLOpc = AArch64::UMULLv2i32_indexed;
951 ReplaceNode(
N, CurDAG->getMachineNode(SMULLOpc, dl,
N->getValueType(0), Ops));
968template<
signed Low,
signed High,
signed Scale>
970 if (!isa<ConstantSDNode>(
N))
973 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
974 if ((MulImm % std::abs(Scale)) == 0) {
975 int64_t RDVLImm = MulImm / Scale;
976 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
987bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
989 unsigned ShiftVal = 0;
1004 Reg =
N.getOperand(0).getOperand(0);
1010 Reg =
N.getOperand(0);
1015 unsigned Opc =
N.getOpcode();
1016 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1033 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1035 return isWorthFolding(
N);
1040bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
1042 unsigned ShiftVal = 0;
1056 Reg =
N.getOperand(0);
1057 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1059 return isWorthFolding(
N);
1068 for (
auto *
Use :
N->uses()) {
1086bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1087 unsigned BW,
unsigned Size,
1094 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1096 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
1102 if (CurDAG->isBaseWithConstantOffset(
N)) {
1103 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1105 int64_t RHSC =
RHS->getSExtValue();
1107 int64_t
Range = 0x1LL << (BW - 1);
1109 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1110 RHSC < (Range << Scale)) {
1111 Base =
N.getOperand(0);
1113 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1116 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl,
MVT::i64);
1125 if ((RHSC & (
Size - 1)) == 0 && RHSC < (Range << Scale)) {
1126 Base =
N.getOperand(0);
1128 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1131 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl,
MVT::i64);
1142 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
1149bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1155 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1157 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
1163 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1164 Base =
N.getOperand(0);
1165 OffImm =
N.getOperand(1);
1174 if (CurDAG->isBaseWithConstantOffset(
N)) {
1175 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1176 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1178 if ((RHSC & (
Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
1179 Base =
N.getOperand(0);
1181 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1184 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl,
MVT::i64);
1192 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1200 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
1209bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1212 if (!CurDAG->isBaseWithConstantOffset(
N))
1214 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1215 int64_t RHSC =
RHS->getSExtValue();
1217 if ((RHSC & (
Size - 1)) == 0 && RHSC >= 0 &&
1220 if (RHSC >= -256 && RHSC < 256) {
1221 Base =
N.getOperand(0);
1223 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1225 Base = CurDAG->getTargetFrameIndex(
1245bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1265 SignExtend = CurDAG->getTargetConstant(0, dl,
MVT::i32);
1271 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1274 return isWorthFolding(
N);
1277bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1289 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1297 if (!isa<MemSDNode>(*UI))
1302 bool IsExtendedRegisterWorthFolding = isWorthFolding(
N);
1305 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1306 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1308 DoShift = CurDAG->getTargetConstant(
true, dl,
MVT::i32);
1313 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1314 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1316 DoShift = CurDAG->getTargetConstant(
true, dl,
MVT::i32);
1321 DoShift = CurDAG->getTargetConstant(
false, dl,
MVT::i32);
1325 if (IsExtendedRegisterWorthFolding &&
1332 if (isWorthFolding(LHS))
1337 if (IsExtendedRegisterWorthFolding &&
1344 if (isWorthFolding(RHS))
1356 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1359 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1361 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1362 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1366bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1381 if (!isa<MemSDNode>(*UI))
1396 if (isa<ConstantSDNode>(RHS)) {
1397 int64_t ImmOff = (int64_t)cast<ConstantSDNode>(RHS)->getZExtValue();
1402 if ((ImmOff %
Size == 0 && ImmOff >= 0 && ImmOff < (0x1000 << Scale)) ||
1408 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL,
MVT::i64, Ops);
1415 bool IsExtendedRegisterWorthFolding = isWorthFolding(
N);
1418 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1419 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1421 DoShift = CurDAG->getTargetConstant(
true,
DL,
MVT::i32);
1426 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1427 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1429 DoShift = CurDAG->getTargetConstant(
true,
DL,
MVT::i32);
1436 SignExtend = CurDAG->getTargetConstant(
false,
DL,
MVT::i32);
1437 DoShift = CurDAG->getTargetConstant(
false,
DL,
MVT::i32);
1443 static const unsigned RegClassIDs[] = {
1444 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1445 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1446 AArch64::dsub2, AArch64::dsub3};
1452 static const unsigned RegClassIDs[] = {
1453 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1454 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1455 AArch64::qsub2, AArch64::qsub3};
1461 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1462 AArch64::ZPR3RegClassID,
1463 AArch64::ZPR4RegClassID};
1464 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1465 AArch64::zsub2, AArch64::zsub3};
1475 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1476 AArch64::ZPR4Mul4RegClassID};
1477 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1478 AArch64::zsub2, AArch64::zsub3};
1483 const unsigned RegClassIDs[],
1484 const unsigned SubRegs[]) {
1487 if (Regs.
size() == 1)
1498 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL,
MVT::i32));
1501 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1507 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL,
MVT::Untyped, Ops);
1511void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1514 EVT VT =
N->getValueType(0);
1516 unsigned ExtOff = isExt;
1519 unsigned Vec0Off = ExtOff + 1;
1521 N->op_begin() + Vec0Off + NumVecs);
1528 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1529 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1532bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1534 if (
LD->isUnindexed())
1536 EVT VT =
LD->getMemoryVT();
1537 EVT DstVT =
N->getValueType(0);
1544 unsigned Opcode = 0;
1547 bool InsertTo64 =
false;
1549 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1552 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1554 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1556 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1565 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1567 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1569 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1578 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1580 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1582 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1589 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1591 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1593 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1595 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1597 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1607 SDNode *Res = CurDAG->getMachineNode(Opcode, dl,
MVT::i64, DstVT,
1612 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1619 SDValue(CurDAG->getMachineNode(
1620 AArch64::SUBREG_TO_REG, dl,
MVT::i64,
1621 CurDAG->getTargetConstant(0, dl,
MVT::i64), LoadedVal,
1626 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1629 CurDAG->RemoveDeadNode(
N);
1633void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1634 unsigned SubRegIdx) {
1636 EVT VT =
N->getValueType(0);
1644 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1646 for (
unsigned i = 0; i < NumVecs; ++i)
1648 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1654 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1656 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1659 CurDAG->RemoveDeadNode(
N);
1662void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1663 unsigned Opc,
unsigned SubRegIdx) {
1665 EVT VT =
N->getValueType(0);
1675 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1683 ReplaceUses(
SDValue(
N, 0), SuperReg);
1685 for (
unsigned i = 0; i < NumVecs; ++i)
1687 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1691 CurDAG->RemoveDeadNode(
N);
1697std::tuple<unsigned, SDValue, SDValue>
1698AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1704 SDValue NewOffset = OldOffset;
1706 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1707 N, OldBase, NewBase, NewOffset);
1711 const bool IsRegReg =
1712 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1715 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1728template <SelectTypeKind Kind>
1774void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1776 EVT VT =
N->getValueType(0);
1778 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1783 for (
unsigned I = 0;
I < 2; ++
I)
1784 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1785 AArch64::psub0 +
I,
DL, VT, SuperReg));
1787 CurDAG->RemoveDeadNode(
N);
1790void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1792 EVT VT =
N->getValueType(0);
1794 SDValue Ops = createZTuple(Regs);
1798 for (
unsigned i = 0; i < NumVecs; ++i)
1799 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1800 AArch64::zsub0 + i,
DL, VT, SuperReg));
1802 CurDAG->RemoveDeadNode(
N);
1806void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1810 assert(Opcode != 0 &&
"Unexpected opcode");
1813 EVT VT =
N->getValueType(0);
1815 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1817 N->op_begin() + StartIdx + NumVecs);
1818 return createZMulTuple(Regs);
1821 SDValue Zdn = GetMultiVecOperand(1);
1825 Zm = GetMultiVecOperand(NumVecs + 1);
1827 Zm =
N->getOperand(NumVecs + 1);
1832 for (
unsigned i = 0; i < NumVecs; ++i)
1833 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1834 AArch64::zsub0 + i,
DL, VT, SuperReg));
1836 CurDAG->RemoveDeadNode(
N);
1840void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1841 unsigned Scale,
unsigned Opc_ri,
1842 unsigned Opc_rr,
bool IsIntr) {
1843 assert(Scale < 4 &&
"Invalid scaling value.");
1845 EVT VT =
N->getValueType(0);
1851 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1852 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1853 CurDAG->getTargetConstant(0,
DL,
MVT::i64), Scale);
1855 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1861 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1863 for (
unsigned i = 0; i < NumVecs; ++i)
1864 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1865 AArch64::zsub0 + i,
DL, VT, SuperReg));
1868 unsigned ChainIdx = NumVecs;
1870 CurDAG->RemoveDeadNode(
N);
1873void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1877 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1880void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
1883 EVT VT =
N->getValueType(0);
1886 SDValue Zd = createZMulTuple(Regs);
1887 SDValue Zn =
N->getOperand(1 + NumVecs);
1888 SDValue Zm =
N->getOperand(2 + NumVecs);
1894 for (
unsigned i = 0; i < NumVecs; ++i)
1895 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1896 AArch64::zsub0 + i,
DL, VT, SuperReg));
1898 CurDAG->RemoveDeadNode(
N);
1929template <
unsigned MaxIdx,
unsigned Scale>
1930void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
1931 unsigned BaseReg,
unsigned Op) {
1932 unsigned TileNum = 0;
1933 if (BaseReg != AArch64::ZA)
1934 TileNum = cast<ConstantSDNode>(
N->getOperand(2))->getZExtValue();
1940 if (BaseReg == AArch64::ZA)
1941 SliceBase =
N->getOperand(2);
1943 SliceBase =
N->getOperand(3);
1945 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
1953 EVT VT =
N->getValueType(0);
1954 for (
unsigned I = 0;
I < NumVecs; ++
I)
1956 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
1959 unsigned ChainIdx = NumVecs;
1961 CurDAG->RemoveDeadNode(
N);
1964void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
1965 unsigned NumOutVecs,
1969 EVT VT =
N->getValueType(0);
1970 unsigned NumInVecs =
N->getNumOperands() - 1;
1974 assert((NumInVecs == 2 || NumInVecs == 4) &&
1975 "Don't know how to handle multi-register input!");
1977 N->op_begin() + 1 + NumInVecs);
1981 for (
unsigned I = 0;
I < NumInVecs;
I++)
1988 for (
unsigned I = 0;
I < NumOutVecs;
I++)
1989 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1990 AArch64::zsub0 +
I,
DL, VT, SuperReg));
1991 CurDAG->RemoveDeadNode(
N);
1994void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
1997 EVT VT =
N->getOperand(2)->getValueType(0);
2004 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2005 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2009 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2014void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2015 unsigned Scale,
unsigned Opc_rr,
2021 SDValue RegSeq = createZTuple(Regs);
2026 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2027 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2028 CurDAG->getTargetConstant(0, dl,
MVT::i64), Scale);
2030 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2034 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2046 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2047 int FI = FINode->getIndex();
2049 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
2056void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2059 EVT VT =
N->getOperand(2)->getValueType(0);
2069 N->getOperand(NumVecs + 1),
2070 N->getOperand(NumVecs + 2),
2072 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2112void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2115 EVT VT =
N->getValueType(0);
2123 WidenVector(*CurDAG));
2130 cast<ConstantSDNode>(
N->getOperand(NumVecs + 2))->getZExtValue();
2132 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl,
MVT::i64),
2133 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2134 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2138 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2139 AArch64::qsub2, AArch64::qsub3 };
2140 for (
unsigned i = 0; i < NumVecs; ++i) {
2141 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2148 CurDAG->RemoveDeadNode(
N);
2151void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2154 EVT VT =
N->getValueType(0);
2162 WidenVector(*CurDAG));
2170 cast<ConstantSDNode>(
N->getOperand(NumVecs + 1))->getZExtValue();
2173 CurDAG->getTargetConstant(LaneNo, dl,
2175 N->getOperand(NumVecs + 2),
2176 N->getOperand(NumVecs + 3),
2178 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2190 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2191 AArch64::qsub2, AArch64::qsub3 };
2192 for (
unsigned i = 0; i < NumVecs; ++i) {
2193 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2203 CurDAG->RemoveDeadNode(
N);
2206void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2209 EVT VT =
N->getOperand(2)->getValueType(0);
2217 WidenVector(*CurDAG));
2222 cast<ConstantSDNode>(
N->getOperand(NumVecs + 2))->getZExtValue();
2224 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl,
MVT::i64),
2225 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2230 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2235void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2238 EVT VT =
N->getOperand(2)->getValueType(0);
2246 WidenVector(*CurDAG));
2254 cast<ConstantSDNode>(
N->getOperand(NumVecs + 1))->getZExtValue();
2256 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl,
MVT::i64),
2257 N->getOperand(NumVecs + 2),
2258 N->getOperand(NumVecs + 3),
2260 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2264 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2271 unsigned &LSB,
unsigned &MSB,
2272 unsigned NumberOfIgnoredLowBits,
2273 bool BiggerPattern) {
2275 "N must be a AND operation to call this function");
2277 EVT VT =
N->getValueType(0);
2283 "Type checking must have been done before calling this function");
2297 const SDNode *Op0 =
N->getOperand(0).getNode();
2301 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2304 if (AndImm & (AndImm + 1))
2307 bool ClampMSB =
false;
2328 }
else if (BiggerPattern) {
2334 Opd0 =
N->getOperand(0);
2340 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2343 <<
": Found large shift immediate, this should not happen\n"));
2349 (VT ==
MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2350 : llvm::countr_one<uint64_t>(AndImm)) -
2357 MSB = MSB > 31 ? 31 : MSB;
2359 Opc = VT ==
MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2364 SDValue &Opd0,
unsigned &Immr,
2368 EVT VT =
N->getValueType(0);
2371 "Type checking must have been done before calling this function");
2375 Op = Op->getOperand(0);
2376 VT = Op->getValueType(0);
2385 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2389 Opc = (VT ==
MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2390 Opd0 = Op.getOperand(0);
2392 Imms = ShiftImm + Width - 1;
2420 Opd0 =
N->getOperand(0).getOperand(0);
2430 Opc =
N->getValueType(0) ==
MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2437 unsigned &Immr,
unsigned &Imms,
2438 bool BiggerPattern) {
2440 "N must be a SHR/SRA operation to call this function");
2442 EVT VT =
N->getValueType(0);
2448 "Type checking must have been done before calling this function");
2458 Opd0 =
N->getOperand(0).getOperand(0);
2465 Opd0 =
N->getOperand(0).getOperand(0);
2469 }
else if (BiggerPattern) {
2473 Opd0 =
N->getOperand(0);
2482 <<
": Found large shift immediate, this should not happen\n"));
2491 "bad amount in shift node!");
2492 int immr = SrlImm - ShlImm;
2497 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2499 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2503bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2506 EVT VT =
N->getValueType(0);
2507 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2519 unsigned Immr = ShiftImm;
2521 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2522 CurDAG->getTargetConstant(Imms, dl, VT)};
2523 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2529bool AArch64DAGToDAGISel::tryHighFPExt(
SDNode *
N) {
2533 SDValue Extract =
N->getOperand(0);
2534 EVT VT =
N->getValueType(0);
2551 auto Opcode = VT ==
MVT::v2f64 ? AArch64::FCVTLv4i32 : AArch64::FCVTLv8i16;
2552 CurDAG->SelectNodeTo(
N, Opcode, VT, Extract.
getOperand(0));
2557 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2558 unsigned NumberOfIgnoredLowBits = 0,
2559 bool BiggerPattern =
false) {
2563 switch (
N->getOpcode()) {
2565 if (!
N->isMachineOpcode())
2570 NumberOfIgnoredLowBits, BiggerPattern);
2579 unsigned NOpc =
N->getMachineOpcode();
2583 case AArch64::SBFMWri:
2584 case AArch64::UBFMWri:
2585 case AArch64::SBFMXri:
2586 case AArch64::UBFMXri:
2588 Opd0 =
N->getOperand(0);
2589 Immr = cast<ConstantSDNode>(
N->getOperand(1).getNode())->getZExtValue();
2590 Imms = cast<ConstantSDNode>(
N->getOperand(2).getNode())->getZExtValue();
2597bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2598 unsigned Opc, Immr, Imms;
2603 EVT VT =
N->getValueType(0);
2608 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT ==
MVT::i32) {
2609 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl,
MVT::i64),
2610 CurDAG->getTargetConstant(Imms, dl,
MVT::i64)};
2613 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2619 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2620 CurDAG->getTargetConstant(Imms, dl, VT)};
2621 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2630 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2632 "i32 or i64 mask type expected!");
2638 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2639 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2662 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
2672 APInt OpUsefulBits(UsefulBits);
2676 OpUsefulBits <<= MSB - Imm + 1;
2681 OpUsefulBits <<= Imm;
2683 OpUsefulBits <<= MSB + 1;
2686 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2692 UsefulBits &= OpUsefulBits;
2698 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
2700 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2708 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2709 APInt Mask(UsefulBits);
2710 Mask.clearAllBits();
2718 Mask.lshrInPlace(ShiftAmt);
2724 Mask.lshrInPlace(ShiftAmt);
2736 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2738 cast<const ConstantSDNode>(Op.getOperand(3).getNode())->getZExtValue();
2740 APInt OpUsefulBits(UsefulBits);
2754 OpUsefulBits <<= Width;
2757 if (Op.getOperand(1) == Orig) {
2759 Mask = ResultUsefulBits & OpUsefulBits;
2763 if (Op.getOperand(0) == Orig)
2765 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2771 OpUsefulBits <<= Width;
2773 OpUsefulBits <<= LSB;
2775 if (Op.getOperand(1) == Orig) {
2777 Mask = ResultUsefulBits & OpUsefulBits;
2778 Mask.lshrInPlace(LSB);
2781 if (Op.getOperand(0) == Orig)
2782 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2799 case AArch64::ANDSWri:
2800 case AArch64::ANDSXri:
2801 case AArch64::ANDWri:
2802 case AArch64::ANDXri:
2806 case AArch64::UBFMWri:
2807 case AArch64::UBFMXri:
2810 case AArch64::ORRWrs:
2811 case AArch64::ORRXrs:
2816 case AArch64::BFMWri:
2817 case AArch64::BFMXri:
2820 case AArch64::STRBBui:
2821 case AArch64::STURBBi:
2827 case AArch64::STRHHui:
2828 case AArch64::STURHHi:
2841 unsigned Bitwidth = Op.getScalarValueSizeInBits();
2843 UsefulBits =
APInt(Bitwidth, 0);
2852 UsersUsefulBits |= UsefulBitsForUse;
2857 UsefulBits &= UsersUsefulBits;
2867 EVT VT = Op.getValueType();
2870 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2873 if (ShlAmount > 0) {
2876 UBFMOpc, dl, VT, Op,
2881 assert(ShlAmount < 0 &&
"expected right shift");
2882 int ShrAmount = -ShlAmount;
2908 bool BiggerPattern,
SDValue &Src,
2909 int &DstLSB,
int &Width) {
2910 EVT VT = Op.getValueType();
2919 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2923 switch (Op.getOpcode()) {
2928 NonZeroBits, Src, DstLSB, Width);
2931 NonZeroBits, Src, DstLSB, Width);
2944 EVT VT = Op.getValueType();
2946 "Caller guarantees VT is one of i32 or i64");
2959 assert((~AndImm & NonZeroBits) == 0 &&
2960 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
2962 SDValue AndOp0 = Op.getOperand(0);
2989 if (!BiggerPattern && !AndOp0.
hasOneUse())
3008 <<
"Found large Width in bit-field-positioning -- this indicates no "
3009 "proper combining / constant folding was performed\n");
3018 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3033 "Op.getNode() should be a SHL node to call this function");
3035 "Op.getNode() should shift ShlImm to call this function");
3038 SDValue Op0 = Op.getOperand(0);
3042 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3066 EVT VT = Op.getValueType();
3068 "Caller guarantees that type is i32 or i64");
3075 if (!BiggerPattern && !Op.hasOneUse())
3084 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3087 Src =
getLeftShift(CurDAG, Op.getOperand(0), ShlImm - DstLSB);
3103 EVT VT =
N->getValueType(0);
3122 if (!
And.hasOneUse() ||
3132 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3139 if ((OrImm & NotKnownZero) != 0) {
3151 unsigned ImmS = Width - 1;
3157 bool IsBFI = LSB != 0;
3162 unsigned OrChunks = 0, BFIChunks = 0;
3163 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3164 if (((OrImm >> Shift) & 0xFFFF) != 0)
3166 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3169 if (BFIChunks > OrChunks)
3175 unsigned MOVIOpc = VT ==
MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3183 unsigned Opc = (VT ==
MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3192 if (!Dst.hasOneUse())
3195 EVT VT = Dst.getValueType();
3197 "Caller should guarantee that VT is one of i32 or i64");
3205 SDValue DstOp0 = Dst.getOperand(0);
3225 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3226 unsigned MaskWidth =
3229 (VT ==
MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3235 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3236 ShiftedOperand =
SDValue(UBFMNode, 0);
3246 ShiftedOperand = Dst.getOperand(0);
3253 ShiftedOperand = Dst.getOperand(0);
3265 const bool BiggerPattern) {
3266 EVT VT =
N->getValueType(0);
3267 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3268 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3269 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3270 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3272 "Expect result type to be i32 or i64 since N is combinable to BFM");
3279 const unsigned OrrOpc = (VT ==
MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3282 if (BiggerPattern) {
3296 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3305 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3367 EVT VT =
N->getValueType(0);
3376 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3377 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3397 for (
int I = 0;
I < 4; ++
I) {
3400 unsigned ImmR, ImmS;
3401 bool BiggerPattern =
I / 2;
3402 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3404 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3410 NumberOfIgnoredLowBits, BiggerPattern)) {
3413 if ((BFXOpc != AArch64::UBFMXri && VT ==
MVT::i64) ||
3414 (BFXOpc != AArch64::UBFMWri && VT ==
MVT::i32))
3419 Width = ImmS - ImmR + 1;
3430 Src, DstLSB, Width)) {
3448 APInt BitsToBeInserted =
3451 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3475 unsigned Opc = (VT ==
MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3508 unsigned ShiftOpc = (VT ==
MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3510 if (Src->hasOneUse() &&
3513 Src = Src->getOperand(0);
3523 unsigned ImmS = Width - 1;
3529 unsigned Opc = (VT ==
MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3537bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3546 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3559bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3563 EVT VT =
N->getValueType(0);
3570 Op0, DstLSB, Width))
3576 unsigned ImmS =
Width - 1;
3579 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3580 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3581 unsigned Opc = (VT ==
MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3582 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3588bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3589 EVT VT =
N->getValueType(0);
3592 switch (
N->getOpcode()) {
3594 Opc = (VT ==
MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3597 Opc = (VT ==
MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3600 Opc = (VT ==
MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3603 Opc = (VT ==
MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3620 SDValue ShiftAmt =
N->getOperand(1);
3640 (Add0Imm %
Size == 0)) {
3647 NegOpc = AArch64::SUBWrr;
3648 ZeroReg = AArch64::WZR;
3651 NegOpc = AArch64::SUBXrr;
3652 ZeroReg = AArch64::XZR;
3655 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3657 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3658 NewShiftAmt =
SDValue(Neg, 0);
3667 NotOpc = AArch64::ORNWrr;
3668 ZeroReg = AArch64::WZR;
3671 NotOpc = AArch64::ORNXrr;
3672 ZeroReg = AArch64::XZR;
3675 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3677 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3678 NewShiftAmt =
SDValue(Not, 0);
3702 AArch64::SUBREG_TO_REG,
DL, VT,
3704 NewShiftAmt =
SDValue(Ext, 0);
3707 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3708 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3713AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3714 unsigned RegWidth) {
3717 FVal = CN->getValueAPF();
3718 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3721 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3725 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3726 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3742 FVal.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact);
3745 if (!IsExact || !
IntVal.isPowerOf2())
return false;
3746 unsigned FBits =
IntVal.logBase2();
3750 if (FBits == 0 || FBits > RegWidth)
return false;
3761 RegString.
split(Fields,
':');
3763 if (Fields.
size() == 1)
3767 &&
"Invalid number of fields in read register string");
3770 bool AllIntFields =
true;
3774 AllIntFields &= !
Field.getAsInteger(10, IntField);
3779 "Unexpected non-integer value in special register string.");
3784 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3785 (Ops[3] << 3) | (Ops[4]);
3792bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3793 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3794 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3799 unsigned Opcode64Bit = AArch64::MRS;
3804 const auto *TheReg =
3806 if (TheReg && TheReg->Readable &&
3807 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3808 Imm = TheReg->Encoding;
3814 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3815 Opcode64Bit = AArch64::ADR;
3823 SDValue InChain =
N->getOperand(0);
3825 if (!ReadIs128Bit) {
3827 {SysRegImm, InChain});
3832 {SysRegImm, InChain});
3844 ReplaceUses(
SDValue(
N, 2), OutChain);
3853bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
3854 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3855 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3860 if (!WriteIs128Bit) {
3866 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
3868 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
3869 "Expected a constant integer expression.");
3870 unsigned Reg = PMapper->Encoding;
3871 uint64_t Immed = cast<ConstantSDNode>(
N->getOperand(2))->getZExtValue();
3872 CurDAG->SelectNodeTo(
3874 CurDAG->getTargetConstant(Immed,
DL,
MVT::i16),
N->getOperand(0));
3880 if (trySelectPState(
3881 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
3882 AArch64::MSRpstateImm4))
3884 if (trySelectPState(
3885 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
3886 AArch64::MSRpstateImm1))
3896 if (TheReg && TheReg->Writeable &&
3897 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3898 Imm = TheReg->Encoding;
3906 SDValue InChain =
N->getOperand(0);
3907 if (!WriteIs128Bit) {
3909 CurDAG->getTargetConstant(Imm,
DL,
MVT::i32),
3910 N->getOperand(2), InChain);
3914 SDNode *Pair = CurDAG->getMachineNode(
3916 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
3919 CurDAG->getTargetConstant(AArch64::sube64,
DL,
MVT::i32),
3921 CurDAG->getTargetConstant(AArch64::subo64,
DL,
MVT::i32)});
3923 CurDAG->SelectNodeTo(
N, AArch64::MSRR,
MVT::Other,
3924 CurDAG->getTargetConstant(Imm,
DL,
MVT::i32),
3932bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3934 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3937 if (Subtarget->hasLSE())
return false;
3940 Opcode = AArch64::CMP_SWAP_8;
3942 Opcode = AArch64::CMP_SWAP_16;
3944 Opcode = AArch64::CMP_SWAP_32;
3946 Opcode = AArch64::CMP_SWAP_64;
3951 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3953 SDNode *CmpSwap = CurDAG->getMachineNode(
3958 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3962 CurDAG->RemoveDeadNode(
N);
3969 if (!isa<ConstantSDNode>(
N))
3981 Shift = CurDAG->getTargetConstant(0,
DL,
MVT::i32);
3989 Shift = CurDAG->getTargetConstant(0,
DL,
MVT::i32);
3994 if (Val <= 65280 && Val % 256 == 0) {
3995 Shift = CurDAG->getTargetConstant(8,
DL,
MVT::i32);
4009 if (!isa<ConstantSDNode>(
N))
4013 int64_t Val = cast<ConstantSDNode>(
N)
4021 Shift = CurDAG->getTargetConstant(0,
DL,
MVT::i32);
4028 if (Val >= -128 && Val <= 127) {
4029 Shift = CurDAG->getTargetConstant(0,
DL,
MVT::i32);
4034 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4035 Shift = CurDAG->getTargetConstant(8,
DL,
MVT::i32);
4036 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL,
MVT::i32);
4047bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4048 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4049 int64_t ImmVal = CNode->getSExtValue();
4051 if (ImmVal >= -128 && ImmVal < 128) {
4060 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4061 uint64_t ImmVal = CNode->getZExtValue();
4071 ImmVal &= 0xFFFFFFFF;
4089 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4090 uint64_t ImmVal = CNode->getZExtValue();
4100 ImmVal |= ImmVal << 8;
4101 ImmVal |= ImmVal << 16;
4102 ImmVal |= ImmVal << 32;
4106 ImmVal |= ImmVal << 16;
4107 ImmVal |= ImmVal << 32;
4110 ImmVal &= 0xFFFFFFFF;
4111 ImmVal |= ImmVal << 32;
4136 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4137 uint64_t ImmVal = CN->getZExtValue();
4144 if (ImmVal >
High) {
4145 if (!AllowSaturation)
4157bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4161 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4167 cast<ConstantSDNode>(IRG_SP->
getOperand(1))->getZExtValue() !=
4168 Intrinsic::aarch64_irg_sp) {
4174 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4175 SDValue FiOp = CurDAG->getTargetFrameIndex(
4177 int TagOffset = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
4179 SDNode *Out = CurDAG->getMachineNode(
4181 {FiOp, CurDAG->getTargetConstant(0,
DL,
MVT::i64),
N->getOperand(2),
4182 CurDAG->getTargetConstant(TagOffset,
DL,
MVT::i64)});
4183 ReplaceNode(
N, Out);
4187void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4188 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4189 "llvm.aarch64.tagp third argument must be an immediate");
4190 if (trySelectStackSlotTagP(
N))
4197 int TagOffset = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
4199 {
N->getOperand(1),
N->getOperand(2)});
4201 {
SDValue(N1, 0),
N->getOperand(2)});
4202 SDNode *N3 = CurDAG->getMachineNode(
4205 CurDAG->getTargetConstant(TagOffset,
DL,
MVT::i64)});
4209bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4213 if (cast<ConstantSDNode>(
N->getOperand(2))->getZExtValue() != 0)
4215 if (!
N->getOperand(0).isUndef())
4219 EVT VT =
N->getValueType(0);
4220 EVT InVT =
N->getOperand(1).getValueType();
4231 "Expected to insert into a packed scalable vector!");
4234 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL,
MVT::i64);
4235 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4236 N->getOperand(1), RC));
4240bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4244 if (cast<ConstantSDNode>(
N->getOperand(1))->getZExtValue() != 0)
4248 EVT VT =
N->getValueType(0);
4249 EVT InVT =
N->getOperand(0).getValueType();
4260 "Expected to extract from a packed scalable vector!");
4263 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL,
MVT::i64);
4264 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4265 N->getOperand(0), RC));
4269void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4271 if (
Node->isMachineOpcode()) {
4273 Node->setNodeId(-1);
4278 EVT VT =
Node->getValueType(0);
4280 switch (
Node->getOpcode()) {
4285 if (SelectCMP_SWAP(
Node))
4291 if (tryReadRegister(
Node))
4297 if (tryWriteRegister(
Node))
4302 if (tryMLAV64LaneV128(
Node))
4309 if (tryIndexedLoad(
Node))
4318 if (tryBitfieldExtractOp(
Node))
4320 if (tryBitfieldInsertInZeroOp(
Node))
4325 if (tryShiftAmountMod(
Node))
4330 if (tryBitfieldExtractOpFromSExt(
Node))
4335 if (tryHighFPExt(
Node))
4340 if (tryBitfieldInsertOp(
Node))
4345 if (trySelectCastScalableToFixedLengthVector(
Node))
4351 if (trySelectCastFixedLengthToScalableVector(
Node))
4360 if (ConstNode->
isZero()) {
4364 ReplaceNode(
Node,
New.getNode());
4369 ReplaceNode(
Node,
New.getNode());
4378 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4381 SDValue TFI = CurDAG->getTargetFrameIndex(
4385 CurDAG->getTargetConstant(Shifter,
DL,
MVT::i32) };
4386 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri,
MVT::i64, Ops);
4390 unsigned IntNo = cast<ConstantSDNode>(
Node->getOperand(1))->getZExtValue();
4394 case Intrinsic::aarch64_ldaxp:
4395 case Intrinsic::aarch64_ldxp: {
4397 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4407 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4408 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4409 ReplaceNode(
Node, Ld);
4412 case Intrinsic::aarch64_stlxp:
4413 case Intrinsic::aarch64_stxp: {
4415 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4423 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4428 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4429 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4431 ReplaceNode(
Node, St);
4434 case Intrinsic::aarch64_neon_ld1x2:
4436 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4439 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4442 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4445 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4448 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4451 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4454 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4457 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4461 case Intrinsic::aarch64_neon_ld1x3:
4463 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4466 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4469 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4472 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4475 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4478 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4481 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4484 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4488 case Intrinsic::aarch64_neon_ld1x4:
4490 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4493 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4496 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4499 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4502 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4505 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4508 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4511 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4515 case Intrinsic::aarch64_neon_ld2:
4517 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4520 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4523 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4526 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4529 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4532 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4535 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4538 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4542 case Intrinsic::aarch64_neon_ld3:
4544 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4547 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4550 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4553 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4556 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4559 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4562 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4565 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4569 case Intrinsic::aarch64_neon_ld4:
4571 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4574 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4577 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4580 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4583 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4586 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4589 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4592 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4596 case Intrinsic::aarch64_neon_ld2r:
4598 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4601 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4604 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4607 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4610 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4613 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4616 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4619 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4623 case Intrinsic::aarch64_neon_ld3r:
4625 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4628 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4631 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4634 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4637 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4640 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4643 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4646 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4650 case Intrinsic::aarch64_neon_ld4r:
4652 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4655 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4658 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4661 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);