22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
47 AArch64DAGToDAGISel() =
delete;
64 std::vector<SDValue> &OutOps)
override;
66 template <
signed Low,
signed High,
signed Scale>
74 return SelectShiftedRegister(
N,
false, Reg, Shift);
77 return SelectShiftedRegister(
N,
true, Reg, Shift);
80 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
83 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
86 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
89 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
92 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
95 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
98 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
101 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
104 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
107 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
110 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
113 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
116 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
119 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
122 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
125 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
128 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
130 template <
unsigned Size,
unsigned Max>
134 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
136 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
137 int64_t
C = CI->getSExtValue();
145 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
152 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
158 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
163 N =
N->getOperand(0);
165 !isa<ConstantSDNode>(
N->getOperand(1)))
167 EVT VT =
N->getValueType(0);
168 EVT LVT =
N->getOperand(0).getValueType();
169 unsigned Index =
N->getConstantOperandVal(1);
173 Res =
N->getOperand(0);
181 EVT VT =
Op.getValueType();
182 unsigned ShtAmt =
N->getConstantOperandVal(1);
189 Op.getOperand(1).getConstantOperandVal(0)
190 <<
Op.getOperand(1).getConstantOperandVal(1));
192 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
194 Op.getOperand(1).getConstantOperandVal(0));
198 if (Imm != 1ULL << (ShtAmt - 1))
201 Res1 =
Op.getOperand(0);
202 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
206 bool SelectDupZeroOrUndef(
SDValue N) {
207 switch(
N->getOpcode()) {
212 auto Opnd0 =
N->getOperand(0);
227 switch(
N->getOpcode()) {
230 auto Opnd0 =
N->getOperand(0);
242 bool SelectDupNegativeZero(
SDValue N) {
243 switch(
N->getOpcode()) {
247 return Const && Const->isZero() && Const->isNegative();
254 template<MVT::SimpleValueType VT>
256 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
259 template <MVT::SimpleValueType VT,
bool Negate>
261 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
264 template <MVT::SimpleValueType VT>
266 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
269 template <MVT::SimpleValueType VT,
bool Invert = false>
271 return SelectSVELogicalImm(
N, VT, Imm, Invert);
274 template <MVT::SimpleValueType VT>
276 return SelectSVEArithImm(
N, VT, Imm);
279 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
281 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
288 EVT EltVT =
N->getValueType(0).getVectorElementType();
289 return SelectSVEShiftImm(
N->getOperand(0), 1,
295 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
297 if (!isa<ConstantSDNode>(
N))
300 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
302 MulImm = 1LL << MulImm;
304 if ((MulImm % std::abs(Scale)) != 0)
308 if ((MulImm >= Min) && (MulImm <= Max)) {
309 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
316 template <
signed Max,
signed Scale>
318 if (!isa<ConstantSDNode>(
N))
321 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
323 if (MulImm >= 0 && MulImm <= Max) {
325 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
332 template <
unsigned BaseReg,
unsigned Max>
334 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
340 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
363 const unsigned SubRegs[]);
365 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
367 bool tryIndexedLoad(
SDNode *
N);
369 bool trySelectStackSlotTagP(
SDNode *
N);
372 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
374 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
376 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
377 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
378 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
379 unsigned Opc_rr,
unsigned Opc_ri,
380 bool IsIntr =
false);
381 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
382 unsigned Scale,
unsigned Opc_ri,
384 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
385 bool IsZmMulti,
unsigned Opcode,
386 bool HasPred =
false);
387 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
388 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
389 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
390 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
391 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
392 bool IsTupleInput,
unsigned Opc);
393 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
395 template <
unsigned MaxIdx,
unsigned Scale>
396 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
398 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
399 unsigned Op,
unsigned MaxIdx,
unsigned Scale,
400 unsigned BaseReg = 0);
403 template <
int64_t Min,
int64_t Max>
407 template <
unsigned Scale>
409 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
412 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
415 template <
unsigned MaxIdx,
unsigned Scale>
420 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
421 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
422 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
423 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
424 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
425 unsigned Opc_rr,
unsigned Opc_ri);
426 std::tuple<unsigned, SDValue, SDValue>
427 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
431 bool tryBitfieldExtractOp(
SDNode *
N);
432 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
433 bool tryBitfieldInsertOp(
SDNode *
N);
434 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
435 bool tryShiftAmountMod(
SDNode *
N);
437 bool tryReadRegister(
SDNode *
N);
438 bool tryWriteRegister(
SDNode *
N);
440 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
441 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
446#include "AArch64GenDAGISel.inc"
454 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
456 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
469 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
470 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
471 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
474 template<
unsigned RegW
idth>
476 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
479 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
481 template<
unsigned RegW
idth>
483 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
489 bool SelectCMP_SWAP(
SDNode *
N);
499 bool AllowSaturation,
SDValue &Imm);
507 bool SelectAllActivePredicate(
SDValue N);
517 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
521char AArch64DAGToDAGISelLegacy::ID = 0;
529 Imm =
C->getZExtValue();
546 return N->getOpcode() == Opc &&
557 return Imm == ImmExpected;
561bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
563 std::vector<SDValue> &OutOps) {
564 switch(ConstraintID) {
567 case InlineAsm::ConstraintCode::m:
568 case InlineAsm::ConstraintCode::o:
569 case InlineAsm::ConstraintCode::Q:
575 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
577 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
578 dl,
Op.getValueType(),
580 OutOps.push_back(NewOp);
596 if (!isa<ConstantSDNode>(
N.getNode()))
599 uint64_t Immed =
N.getNode()->getAsZExtVal();
602 if (Immed >> 12 == 0) {
604 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
612 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
613 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
626 if (!isa<ConstantSDNode>(
N.getNode()))
630 uint64_t Immed =
N.getNode()->getAsZExtVal();
638 if (
N.getValueType() == MVT::i32)
641 Immed = ~Immed + 1ULL;
642 if (Immed & 0xFFFFFFFFFF000000ULL)
645 Immed &= 0xFFFFFFULL;
646 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
653 switch (
N.getOpcode()) {
672 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
675 unsigned ShiftVal = CSD->getZExtValue();
684 if (!isa<MemSDNode>(*UI))
686 if (!isa<MemSDNode>(*UII))
693bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
696 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
701 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
723bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
725 EVT VT =
N.getValueType();
726 if (VT != MVT::i32 && VT != MVT::i64)
729 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
735 unsigned LHSOpcode =
LHS->getOpcode();
749 unsigned LowZBits, MaskLen;
753 unsigned BitWidth =
N.getValueSizeInBits();
760 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
763 NewShiftC = LowZBits - ShiftAmtC;
764 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
770 NewShiftC = LowZBits + ShiftAmtC;
783 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
785 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
789 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
791 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
792 NewShiftAmt, BitWidthMinus1),
795 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
807 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
809 SrcVT =
N.getOperand(0).getValueType();
811 if (!IsLoadStore && SrcVT == MVT::i8)
813 else if (!IsLoadStore && SrcVT == MVT::i16)
815 else if (SrcVT == MVT::i32)
817 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
822 EVT SrcVT =
N.getOperand(0).getValueType();
823 if (!IsLoadStore && SrcVT == MVT::i8)
825 else if (!IsLoadStore && SrcVT == MVT::i16)
827 else if (SrcVT == MVT::i32)
829 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
857bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
860 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
865 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
866 V.getConstantOperandVal(1) <= 4 &&
879bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
881 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
891 unsigned BitSize =
N.getValueSizeInBits();
892 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
895 Reg =
N.getOperand(0);
896 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
897 return isWorthFoldingALU(
N,
true);
908 if (
N.getValueType() == MVT::i32)
916template<
signed Low,
signed High,
signed Scale>
918 if (!isa<ConstantSDNode>(
N))
921 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
922 if ((MulImm % std::abs(Scale)) == 0) {
923 int64_t RDVLImm = MulImm / Scale;
924 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
925 Imm = CurDAG->getTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
935bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
937 unsigned ShiftVal = 0;
952 Reg =
N.getOperand(0).getOperand(0);
958 Reg =
N.getOperand(0);
963 unsigned Opc =
N.getOpcode();
964 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
981 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
983 return isWorthFoldingALU(
N);
988bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
990 unsigned ShiftVal = 0;
1004 Reg =
N.getOperand(0);
1005 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1007 return isWorthFoldingALU(
N);
1016 for (
auto *
Use :
N->uses()) {
1043bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1044 unsigned BW,
unsigned Size,
1051 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1053 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1059 if (CurDAG->isBaseWithConstantOffset(
N)) {
1060 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1062 int64_t RHSC =
RHS->getSExtValue();
1064 int64_t
Range = 0x1LL << (BW - 1);
1066 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1067 RHSC < (
Range << Scale)) {
1068 Base =
N.getOperand(0);
1070 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1073 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1082 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1083 Base =
N.getOperand(0);
1085 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1088 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1099 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1106bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1112 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1114 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1120 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1121 Base =
N.getOperand(0);
1122 OffImm =
N.getOperand(1);
1131 if (CurDAG->isBaseWithConstantOffset(
N)) {
1132 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1133 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1136 Base =
N.getOperand(0);
1138 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1141 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1149 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1157 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1166bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1169 if (!CurDAG->isBaseWithConstantOffset(
N))
1171 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1172 int64_t RHSC =
RHS->getSExtValue();
1173 if (RHSC >= -256 && RHSC < 256) {
1174 Base =
N.getOperand(0);
1176 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1178 Base = CurDAG->getTargetFrameIndex(
1181 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1191 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1198bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1218 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1224 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1227 return isWorthFoldingAddr(
N,
Size);
1230bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1242 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1250 if (!isa<MemSDNode>(*UI))
1255 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1258 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1259 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1261 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1266 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1267 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1269 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1274 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1278 if (IsExtendedRegisterWorthFolding &&
1285 if (isWorthFoldingAddr(LHS,
Size))
1290 if (IsExtendedRegisterWorthFolding &&
1297 if (isWorthFoldingAddr(RHS,
Size))
1309 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1312 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1314 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1315 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1319bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1334 if (!isa<MemSDNode>(*UI))
1349 if (isa<ConstantSDNode>(RHS)) {
1350 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1360 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1363 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1367 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1370 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1371 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1373 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1378 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1379 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1381 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1388 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1389 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1395 static const unsigned RegClassIDs[] = {
1396 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1397 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1398 AArch64::dsub2, AArch64::dsub3};
1404 static const unsigned RegClassIDs[] = {
1405 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1406 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1407 AArch64::qsub2, AArch64::qsub3};
1413 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1414 AArch64::ZPR3RegClassID,
1415 AArch64::ZPR4RegClassID};
1416 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1417 AArch64::zsub2, AArch64::zsub3};
1427 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1428 AArch64::ZPR4Mul4RegClassID};
1429 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1430 AArch64::zsub2, AArch64::zsub3};
1435 const unsigned RegClassIDs[],
1436 const unsigned SubRegs[]) {
1439 if (Regs.
size() == 1)
1450 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1453 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1455 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1459 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1463void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1466 EVT VT =
N->getValueType(0);
1468 unsigned ExtOff = isExt;
1471 unsigned Vec0Off = ExtOff + 1;
1473 N->op_begin() + Vec0Off + NumVecs);
1480 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1481 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1484bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1486 if (
LD->isUnindexed())
1488 EVT VT =
LD->getMemoryVT();
1489 EVT DstVT =
N->getValueType(0);
1496 unsigned Opcode = 0;
1499 bool InsertTo64 =
false;
1501 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1502 else if (VT == MVT::i32) {
1504 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1506 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1508 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1514 }
else if (VT == MVT::i16) {
1516 if (DstVT == MVT::i64)
1517 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1519 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1521 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1522 InsertTo64 = DstVT == MVT::i64;
1527 }
else if (VT == MVT::i8) {
1529 if (DstVT == MVT::i64)
1530 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1532 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1534 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1535 InsertTo64 = DstVT == MVT::i64;
1540 }
else if (VT == MVT::f16) {
1541 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1542 }
else if (VT == MVT::bf16) {
1543 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1544 }
else if (VT == MVT::f32) {
1545 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1547 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1549 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1557 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1559 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1564 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1569 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1571 SDValue(CurDAG->getMachineNode(
1572 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1573 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1578 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1581 CurDAG->RemoveDeadNode(
N);
1585void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1586 unsigned SubRegIdx) {
1588 EVT VT =
N->getValueType(0);
1594 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1596 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1598 for (
unsigned i = 0; i < NumVecs; ++i)
1600 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1606 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1608 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1611 CurDAG->RemoveDeadNode(
N);
1614void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1615 unsigned Opc,
unsigned SubRegIdx) {
1617 EVT VT =
N->getValueType(0);
1624 const EVT ResTys[] = {MVT::i64,
1625 MVT::Untyped, MVT::Other};
1627 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1635 ReplaceUses(
SDValue(
N, 0), SuperReg);
1637 for (
unsigned i = 0; i < NumVecs; ++i)
1639 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1643 CurDAG->RemoveDeadNode(
N);
1649std::tuple<unsigned, SDValue, SDValue>
1650AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1656 SDValue NewOffset = OldOffset;
1658 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1659 N, OldBase, NewBase, NewOffset);
1663 const bool IsRegReg =
1664 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1667 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1680template <SelectTypeKind Kind>
1692 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1697 if (EltVT != MVT::i1)
1701 if (EltVT == MVT::bf16)
1703 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1733void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1736 if (
Imm->getZExtValue() > 1)
1740 EVT VT =
N->getValueType(0);
1741 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1742 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1745 for (
unsigned I = 0;
I < 2; ++
I)
1746 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1747 AArch64::psub0 +
I,
DL, VT, SuperReg));
1749 CurDAG->RemoveDeadNode(
N);
1752void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1754 EVT VT =
N->getValueType(0);
1756 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1758 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1761 for (
unsigned I = 0;
I < 2; ++
I)
1762 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1763 AArch64::psub0 +
I,
DL, VT, SuperReg));
1765 CurDAG->RemoveDeadNode(
N);
1768void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1770 EVT VT =
N->getValueType(0);
1772 SDValue Ops = createZTuple(Regs);
1776 for (
unsigned i = 0; i < NumVecs; ++i)
1777 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1778 AArch64::zsub0 + i,
DL, VT, SuperReg));
1780 CurDAG->RemoveDeadNode(
N);
1783void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1788 assert(Opcode != 0 &&
"Unexpected opcode");
1791 EVT VT =
N->getValueType(0);
1792 unsigned FirstVecIdx = HasPred ? 2 : 1;
1794 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1796 N->op_begin() + StartIdx + NumVecs);
1797 return createZMulTuple(Regs);
1800 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1804 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1806 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1810 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1811 N->getOperand(1), Zdn, Zm);
1813 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1815 for (
unsigned i = 0; i < NumVecs; ++i)
1816 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1817 AArch64::zsub0 + i,
DL, VT, SuperReg));
1819 CurDAG->RemoveDeadNode(
N);
1822void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1823 unsigned Scale,
unsigned Opc_ri,
1824 unsigned Opc_rr,
bool IsIntr) {
1825 assert(Scale < 5 &&
"Invalid scaling value.");
1827 EVT VT =
N->getValueType(0);
1833 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1834 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1835 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1837 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1841 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1843 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1845 for (
unsigned i = 0; i < NumVecs; ++i)
1846 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1847 AArch64::zsub0 + i,
DL, VT, SuperReg));
1850 unsigned ChainIdx = NumVecs;
1852 CurDAG->RemoveDeadNode(
N);
1855void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1860 assert(Scale < 4 &&
"Invalid scaling value.");
1862 EVT VT =
N->getValueType(0);
1870 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1876 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1878 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1880 for (
unsigned i = 0; i < NumVecs; ++i)
1881 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1882 AArch64::zsub0 + i,
DL, VT, SuperReg));
1885 unsigned ChainIdx = NumVecs;
1887 CurDAG->RemoveDeadNode(
N);
1890void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1892 if (
N->getValueType(0) != MVT::nxv4f32)
1894 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1897void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
1898 unsigned NumOutVecs,
1901 if (
Imm->getZExtValue() > MaxImm)
1905 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
1909 EVT VT =
Node->getValueType(0);
1912 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
1915 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
1916 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
1917 AArch64::zsub0 +
I,
DL, VT, SuperReg));
1920 unsigned ChainIdx = NumOutVecs;
1922 CurDAG->RemoveDeadNode(
Node);
1925void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
1928 EVT VT =
N->getValueType(0);
1931 SDValue Zd = createZMulTuple(Regs);
1932 SDValue Zn =
N->getOperand(1 + NumVecs);
1933 SDValue Zm =
N->getOperand(2 + NumVecs);
1939 for (
unsigned i = 0; i < NumVecs; ++i)
1940 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1941 AArch64::zsub0 + i,
DL, VT, SuperReg));
1943 CurDAG->RemoveDeadNode(
N);
1973template <
unsigned MaxIdx,
unsigned Scale>
1974void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
1975 unsigned BaseReg,
unsigned Op) {
1976 unsigned TileNum = 0;
1977 if (BaseReg != AArch64::ZA)
1978 TileNum =
N->getConstantOperandVal(2);
1984 if (BaseReg == AArch64::ZA)
1985 SliceBase =
N->getOperand(2);
1987 SliceBase =
N->getOperand(3);
1989 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
1995 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
1997 EVT VT =
N->getValueType(0);
1998 for (
unsigned I = 0;
I < NumVecs; ++
I)
2000 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2003 unsigned ChainIdx = NumVecs;
2005 CurDAG->RemoveDeadNode(
N);
2008void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
2009 unsigned Op,
unsigned MaxIdx,
2010 unsigned Scale,
unsigned BaseReg) {
2014 SDValue SliceBase =
N->getOperand(2);
2015 if (BaseReg != AArch64::ZA)
2016 SliceBase =
N->getOperand(3);
2019 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2026 if (BaseReg != AArch64::ZA )
2031 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2033 EVT VT =
N->getValueType(0);
2034 for (
unsigned I = 0;
I < NumVecs; ++
I)
2036 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2040 unsigned ChainIdx = NumVecs;
2042 CurDAG->RemoveDeadNode(
N);
2045void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2046 unsigned NumOutVecs,
2050 EVT VT =
N->getValueType(0);
2051 unsigned NumInVecs =
N->getNumOperands() - 1;
2055 assert((NumInVecs == 2 || NumInVecs == 4) &&
2056 "Don't know how to handle multi-register input!");
2058 N->op_begin() + 1 + NumInVecs);
2062 for (
unsigned I = 0;
I < NumInVecs;
I++)
2066 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2069 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2070 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2071 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2072 CurDAG->RemoveDeadNode(
N);
2075void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2078 EVT VT =
N->getOperand(2)->getValueType(0);
2085 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2086 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2090 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2095void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2096 unsigned Scale,
unsigned Opc_rr,
2102 SDValue RegSeq = createZTuple(Regs);
2107 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2108 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2109 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2111 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2115 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2127 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2128 int FI = FINode->getIndex();
2130 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2137void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2140 EVT VT =
N->getOperand(2)->getValueType(0);
2141 const EVT ResTys[] = {MVT::i64,
2150 N->getOperand(NumVecs + 1),
2151 N->getOperand(NumVecs + 2),
2153 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2193void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2196 EVT VT =
N->getValueType(0);
2204 WidenVector(*CurDAG));
2208 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2210 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2212 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2213 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2214 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2218 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2219 AArch64::qsub2, AArch64::qsub3 };
2220 for (
unsigned i = 0; i < NumVecs; ++i) {
2221 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2228 CurDAG->RemoveDeadNode(
N);
2231void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2234 EVT VT =
N->getValueType(0);
2242 WidenVector(*CurDAG));
2246 const EVT ResTys[] = {MVT::i64,
2249 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2252 CurDAG->getTargetConstant(LaneNo, dl,
2254 N->getOperand(NumVecs + 2),
2255 N->getOperand(NumVecs + 3),
2257 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2269 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2270 AArch64::qsub2, AArch64::qsub3 };
2271 for (
unsigned i = 0; i < NumVecs; ++i) {
2272 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2282 CurDAG->RemoveDeadNode(
N);
2285void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2288 EVT VT =
N->getOperand(2)->getValueType(0);
2296 WidenVector(*CurDAG));
2300 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2302 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2303 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2304 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2308 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2313void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2316 EVT VT =
N->getOperand(2)->getValueType(0);
2324 WidenVector(*CurDAG));
2328 const EVT ResTys[] = {MVT::i64,
2331 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2333 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2334 N->getOperand(NumVecs + 2),
2335 N->getOperand(NumVecs + 3),
2337 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2341 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2348 unsigned &LSB,
unsigned &MSB,
2349 unsigned NumberOfIgnoredLowBits,
2350 bool BiggerPattern) {
2352 "N must be a AND operation to call this function");
2354 EVT VT =
N->getValueType(0);
2359 assert((VT == MVT::i32 || VT == MVT::i64) &&
2360 "Type checking must have been done before calling this function");
2374 const SDNode *Op0 =
N->getOperand(0).getNode();
2378 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2381 if (AndImm & (AndImm + 1))
2384 bool ClampMSB =
false;
2404 ClampMSB = (VT == MVT::i32);
2405 }
else if (BiggerPattern) {
2411 Opd0 =
N->getOperand(0);
2417 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2420 <<
": Found large shift immediate, this should not happen\n"));
2426 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2427 : llvm::countr_one<uint64_t>(AndImm)) -
2434 MSB = MSB > 31 ? 31 : MSB;
2436 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2441 SDValue &Opd0,
unsigned &Immr,
2445 EVT VT =
N->getValueType(0);
2447 assert((VT == MVT::i32 || VT == MVT::i64) &&
2448 "Type checking must have been done before calling this function");
2452 Op =
Op->getOperand(0);
2453 VT =
Op->getValueType(0);
2462 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2466 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2467 Opd0 =
Op.getOperand(0);
2469 Imms = ShiftImm + Width - 1;
2497 Opd0 =
N->getOperand(0).getOperand(0);
2507 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2514 unsigned &Immr,
unsigned &Imms,
2515 bool BiggerPattern) {
2517 "N must be a SHR/SRA operation to call this function");
2519 EVT VT =
N->getValueType(0);
2524 assert((VT == MVT::i32 || VT == MVT::i64) &&
2525 "Type checking must have been done before calling this function");
2535 Opd0 =
N->getOperand(0).getOperand(0);
2536 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2542 Opd0 =
N->getOperand(0).getOperand(0);
2545 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2546 }
else if (BiggerPattern) {
2550 Opd0 =
N->getOperand(0);
2559 <<
": Found large shift immediate, this should not happen\n"));
2568 "bad amount in shift node!");
2569 int immr = SrlImm - ShlImm;
2574 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2576 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2580bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2583 EVT VT =
N->getValueType(0);
2584 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2585 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2596 unsigned Immr = ShiftImm;
2598 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2599 CurDAG->getTargetConstant(Imms, dl, VT)};
2600 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2605 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2606 unsigned NumberOfIgnoredLowBits = 0,
2607 bool BiggerPattern =
false) {
2608 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2611 switch (
N->getOpcode()) {
2613 if (!
N->isMachineOpcode())
2618 NumberOfIgnoredLowBits, BiggerPattern);
2627 unsigned NOpc =
N->getMachineOpcode();
2631 case AArch64::SBFMWri:
2632 case AArch64::UBFMWri:
2633 case AArch64::SBFMXri:
2634 case AArch64::UBFMXri:
2636 Opd0 =
N->getOperand(0);
2637 Immr =
N->getConstantOperandVal(1);
2638 Imms =
N->getConstantOperandVal(2);
2645bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2646 unsigned Opc, Immr, Imms;
2651 EVT VT =
N->getValueType(0);
2656 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2657 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2658 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2660 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2661 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2667 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2668 CurDAG->getTargetConstant(Imms, dl, VT)};
2669 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2678 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2679 assert((VT == MVT::i32 || VT == MVT::i64) &&
2680 "i32 or i64 mask type expected!");
2686 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2687 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2710 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2720 APInt OpUsefulBits(UsefulBits);
2724 OpUsefulBits <<= MSB - Imm + 1;
2729 OpUsefulBits <<= Imm;
2731 OpUsefulBits <<= MSB + 1;
2734 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2740 UsefulBits &= OpUsefulBits;
2746 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2748 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2756 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2757 APInt Mask(UsefulBits);
2758 Mask.clearAllBits();
2766 Mask.lshrInPlace(ShiftAmt);
2772 Mask.lshrInPlace(ShiftAmt);
2784 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2786 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2788 APInt OpUsefulBits(UsefulBits);
2802 OpUsefulBits <<= Width;
2805 if (
Op.getOperand(1) == Orig) {
2807 Mask = ResultUsefulBits & OpUsefulBits;
2811 if (
Op.getOperand(0) == Orig)
2813 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2819 OpUsefulBits <<= Width;
2821 OpUsefulBits <<= LSB;
2823 if (
Op.getOperand(1) == Orig) {
2825 Mask = ResultUsefulBits & OpUsefulBits;
2826 Mask.lshrInPlace(LSB);
2829 if (
Op.getOperand(0) == Orig)
2830 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2847 case AArch64::ANDSWri:
2848 case AArch64::ANDSXri:
2849 case AArch64::ANDWri:
2850 case AArch64::ANDXri:
2854 case AArch64::UBFMWri:
2855 case AArch64::UBFMXri:
2858 case AArch64::ORRWrs:
2859 case AArch64::ORRXrs:
2864 case AArch64::BFMWri:
2865 case AArch64::BFMXri:
2868 case AArch64::STRBBui:
2869 case AArch64::STURBBi:
2875 case AArch64::STRHHui:
2876 case AArch64::STURHHi:
2889 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
2891 UsefulBits =
APInt(Bitwidth, 0);
2900 UsersUsefulBits |= UsefulBitsForUse;
2905 UsefulBits &= UsersUsefulBits;
2915 EVT VT =
Op.getValueType();
2918 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2921 if (ShlAmount > 0) {
2924 UBFMOpc, dl, VT,
Op,
2929 assert(ShlAmount < 0 &&
"expected right shift");
2930 int ShrAmount = -ShlAmount;
2956 bool BiggerPattern,
SDValue &Src,
2957 int &DstLSB,
int &Width) {
2958 EVT VT =
Op.getValueType();
2967 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2971 switch (
Op.getOpcode()) {
2976 NonZeroBits, Src, DstLSB, Width);
2979 NonZeroBits, Src, DstLSB, Width);
2992 EVT VT =
Op.getValueType();
2993 assert((VT == MVT::i32 || VT == MVT::i64) &&
2994 "Caller guarantees VT is one of i32 or i64");
3007 assert((~AndImm & NonZeroBits) == 0 &&
3008 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3037 if (!BiggerPattern && !AndOp0.
hasOneUse())
3056 <<
"Found large Width in bit-field-positioning -- this indicates no "
3057 "proper combining / constant folding was performed\n");
3066 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3081 "Op.getNode() should be a SHL node to call this function");
3083 "Op.getNode() should shift ShlImm to call this function");
3090 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3114 EVT VT =
Op.getValueType();
3115 assert((VT == MVT::i32 || VT == MVT::i64) &&
3116 "Caller guarantees that type is i32 or i64");
3123 if (!BiggerPattern && !
Op.hasOneUse())
3132 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3140 assert(VT == MVT::i32 || VT == MVT::i64);
3151 EVT VT =
N->getValueType(0);
3152 if (VT != MVT::i32 && VT != MVT::i64)
3170 if (!
And.hasOneUse() ||
3180 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3187 if ((OrImm & NotKnownZero) != 0) {
3199 unsigned ImmS = Width - 1;
3205 bool IsBFI = LSB != 0;
3210 unsigned OrChunks = 0, BFIChunks = 0;
3211 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3212 if (((OrImm >> Shift) & 0xFFFF) != 0)
3214 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3217 if (BFIChunks > OrChunks)
3223 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3231 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3240 if (!Dst.hasOneUse())
3243 EVT VT = Dst.getValueType();
3244 assert((VT == MVT::i32 || VT == MVT::i64) &&
3245 "Caller should guarantee that VT is one of i32 or i64");
3253 SDValue DstOp0 = Dst.getOperand(0);
3273 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3274 unsigned MaskWidth =
3277 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3283 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3284 ShiftedOperand =
SDValue(UBFMNode, 0);
3294 ShiftedOperand = Dst.getOperand(0);
3301 ShiftedOperand = Dst.getOperand(0);
3313 const bool BiggerPattern) {
3314 EVT VT =
N->getValueType(0);
3315 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3316 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3317 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3318 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3319 assert((VT == MVT::i32 || VT == MVT::i64) &&
3320 "Expect result type to be i32 or i64 since N is combinable to BFM");
3327 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3330 if (BiggerPattern) {
3344 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3353 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3415 EVT VT =
N->getValueType(0);
3416 if (VT != MVT::i32 && VT != MVT::i64)
3424 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3425 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3445 for (
int I = 0;
I < 4; ++
I) {
3448 unsigned ImmR, ImmS;
3449 bool BiggerPattern =
I / 2;
3450 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3452 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3458 NumberOfIgnoredLowBits, BiggerPattern)) {
3461 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3462 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3467 Width = ImmS - ImmR + 1;
3478 Src, DstLSB, Width)) {
3486 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3496 APInt BitsToBeInserted =
3499 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3523 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3556 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3558 if (Src->hasOneUse() &&
3561 Src = Src->getOperand(0);
3571 unsigned ImmS = Width - 1;
3577 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3585bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3594 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3607bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3611 EVT VT =
N->getValueType(0);
3612 if (VT != MVT::i32 && VT != MVT::i64)
3618 Op0, DstLSB, Width))
3624 unsigned ImmS = Width - 1;
3627 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3628 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3629 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3630 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3636bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3637 EVT VT =
N->getValueType(0);
3640 switch (
N->getOpcode()) {
3642 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3645 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3648 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3651 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3659 if (VT == MVT::i32) {
3662 }
else if (VT == MVT::i64) {
3668 SDValue ShiftAmt =
N->getOperand(1);
3688 (Add0Imm %
Size == 0)) {
3694 if (SubVT == MVT::i32) {
3695 NegOpc = AArch64::SUBWrr;
3696 ZeroReg = AArch64::WZR;
3698 assert(SubVT == MVT::i64);
3699 NegOpc = AArch64::SUBXrr;
3700 ZeroReg = AArch64::XZR;
3703 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3705 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3706 NewShiftAmt =
SDValue(Neg, 0);
3714 if (SubVT == MVT::i32) {
3715 NotOpc = AArch64::ORNWrr;
3716 ZeroReg = AArch64::WZR;
3718 assert(SubVT == MVT::i64);
3719 NotOpc = AArch64::ORNXrr;
3720 ZeroReg = AArch64::XZR;
3723 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3725 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3726 NewShiftAmt =
SDValue(Not, 0);
3747 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3748 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3750 AArch64::SUBREG_TO_REG,
DL, VT,
3751 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3752 NewShiftAmt =
SDValue(Ext, 0);
3755 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3756 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3763 bool isReciprocal) {
3766 FVal = CN->getValueAPF();
3767 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3770 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3774 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3775 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3798 if (!IsExact || !IntVal.isPowerOf2())
3800 unsigned FBits = IntVal.logBase2();
3804 if (FBits == 0 || FBits > RegWidth)
return false;
3810bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3811 unsigned RegWidth) {
3816bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3818 unsigned RegWidth) {
3828 RegString.
split(Fields,
':');
3830 if (Fields.
size() == 1)
3834 &&
"Invalid number of fields in read register string");
3837 bool AllIntFields =
true;
3841 AllIntFields &= !
Field.getAsInteger(10, IntField);
3846 "Unexpected non-integer value in special register string.");
3851 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3852 (Ops[3] << 3) | (Ops[4]);
3859bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3860 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3861 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3866 unsigned Opcode64Bit = AArch64::MRS;
3871 const auto *TheReg =
3873 if (TheReg && TheReg->Readable &&
3874 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3875 Imm = TheReg->Encoding;
3881 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3882 Opcode64Bit = AArch64::ADR;
3890 SDValue InChain =
N->getOperand(0);
3891 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
3892 if (!ReadIs128Bit) {
3893 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
3894 {SysRegImm, InChain});
3898 {MVT::Untyped , MVT::Other },
3899 {SysRegImm, InChain});
3903 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
3905 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
3911 ReplaceUses(
SDValue(
N, 2), OutChain);
3920bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
3921 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3922 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3927 if (!WriteIs128Bit) {
3933 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
3935 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
3936 "Expected a constant integer expression.");
3937 unsigned Reg = PMapper->Encoding;
3938 uint64_t Immed =
N->getConstantOperandVal(2);
3939 CurDAG->SelectNodeTo(
3940 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
3941 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
3947 if (trySelectPState(
3948 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
3949 AArch64::MSRpstateImm4))
3951 if (trySelectPState(
3952 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
3953 AArch64::MSRpstateImm1))
3963 if (TheReg && TheReg->Writeable &&
3964 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3965 Imm = TheReg->Encoding;
3973 SDValue InChain =
N->getOperand(0);
3974 if (!WriteIs128Bit) {
3975 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
3976 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3977 N->getOperand(2), InChain);
3981 SDNode *Pair = CurDAG->getMachineNode(
3982 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
3983 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
3986 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
3988 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
3990 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
3991 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3999bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
4001 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
4004 if (Subtarget->hasLSE())
return false;
4006 if (MemTy == MVT::i8)
4007 Opcode = AArch64::CMP_SWAP_8;
4008 else if (MemTy == MVT::i16)
4009 Opcode = AArch64::CMP_SWAP_16;
4010 else if (MemTy == MVT::i32)
4011 Opcode = AArch64::CMP_SWAP_32;
4012 else if (MemTy == MVT::i64)
4013 Opcode = AArch64::CMP_SWAP_64;
4017 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4018 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4020 SDNode *CmpSwap = CurDAG->getMachineNode(
4022 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4025 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
4029 CurDAG->RemoveDeadNode(
N);
4036 if (!isa<ConstantSDNode>(
N))
4048 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4049 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4056 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4057 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4061 if (Val <= 65280 && Val % 256 == 0) {
4062 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4063 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4074bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4077 if (!isa<ConstantSDNode>(
N))
4081 int64_t Val = cast<ConstantSDNode>(
N)
4098 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4099 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4106 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4107 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4111 if (Val <= 65280 && Val % 256 == 0) {
4112 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4113 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4126 if (!isa<ConstantSDNode>(
N))
4130 int64_t Val = cast<ConstantSDNode>(
N)
4138 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4139 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4145 if (Val >= -128 && Val <= 127) {
4146 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4147 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4151 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4152 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4153 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4164bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4165 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4166 int64_t ImmVal = CNode->getSExtValue();
4168 if (ImmVal >= -128 && ImmVal < 128) {
4169 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4177 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4178 uint64_t ImmVal = CNode->getZExtValue();
4188 ImmVal &= 0xFFFFFFFF;
4197 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4206 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4207 uint64_t ImmVal = CNode->getZExtValue();
4217 ImmVal |= ImmVal << 8;
4218 ImmVal |= ImmVal << 16;
4219 ImmVal |= ImmVal << 32;
4223 ImmVal |= ImmVal << 16;
4224 ImmVal |= ImmVal << 32;
4227 ImmVal &= 0xFFFFFFFF;
4228 ImmVal |= ImmVal << 32;
4238 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4253 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4254 uint64_t ImmVal = CN->getZExtValue();
4261 if (ImmVal >
High) {
4262 if (!AllowSaturation)
4267 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4274bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4278 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4290 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4291 SDValue FiOp = CurDAG->getTargetFrameIndex(
4293 int TagOffset =
N->getConstantOperandVal(3);
4295 SDNode *Out = CurDAG->getMachineNode(
4296 AArch64::TAGPstack,
DL, MVT::i64,
4297 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4298 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4299 ReplaceNode(
N, Out);
4303void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4304 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4305 "llvm.aarch64.tagp third argument must be an immediate");
4306 if (trySelectStackSlotTagP(
N))
4313 int TagOffset =
N->getConstantOperandVal(3);
4314 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4315 {
N->getOperand(1),
N->getOperand(2)});
4316 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4317 {
SDValue(N1, 0),
N->getOperand(2)});
4318 SDNode *N3 = CurDAG->getMachineNode(
4319 AArch64::ADDG,
DL, MVT::i64,
4320 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4321 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4325bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4329 if (
N->getConstantOperandVal(2) != 0)
4331 if (!
N->getOperand(0).isUndef())
4335 EVT VT =
N->getValueType(0);
4336 EVT InVT =
N->getOperand(1).getValueType();
4347 "Expected to insert into a packed scalable vector!");
4350 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4351 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4352 N->getOperand(1), RC));
4356bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4360 if (
N->getConstantOperandVal(1) != 0)
4364 EVT VT =
N->getValueType(0);
4365 EVT InVT =
N->getOperand(0).getValueType();
4376 "Expected to extract from a packed scalable vector!");
4379 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4380 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4381 N->getOperand(0), RC));
4385bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4390 EVT VT =
N->getValueType(0);
4402 (Subtarget->hasSVE2() ||
4403 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4412 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4413 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4420 APInt ShlAmt, ShrAmt;
4433 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4434 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4435 AArch64::XAR_ZZZI_D})) {
4436 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4442 if (!Subtarget->hasSHA3())
4464 if (ShAmt + HsAmt != 64)
4468 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4473void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4475 if (
Node->isMachineOpcode()) {
4477 Node->setNodeId(-1);
4482 EVT VT =
Node->getValueType(0);
4484 switch (
Node->getOpcode()) {
4489 if (SelectCMP_SWAP(
Node))
4495 if (tryReadRegister(
Node))
4501 if (tryWriteRegister(
Node))
4508 if (tryIndexedLoad(
Node))
4517 if (tryBitfieldExtractOp(
Node))
4519 if (tryBitfieldInsertInZeroOp(
Node))
4524 if (tryShiftAmountMod(
Node))
4529 if (tryBitfieldExtractOpFromSExt(
Node))
4534 if (tryBitfieldInsertOp(
Node))
4536 if (trySelectXAR(
Node))
4541 if (trySelectCastScalableToFixedLengthVector(
Node))
4547 if (trySelectCastFixedLengthToScalableVector(
Node))
4556 if (ConstNode->
isZero()) {
4557 if (VT == MVT::i32) {
4559 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4560 ReplaceNode(
Node,
New.getNode());
4562 }
else if (VT == MVT::i64) {
4564 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4565 ReplaceNode(
Node,
New.getNode());
4574 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4577 SDValue TFI = CurDAG->getTargetFrameIndex(
4580 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4581 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4582 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4586 unsigned IntNo =
Node->getConstantOperandVal(1);
4590 case Intrinsic::aarch64_gcsss: {
4594 SDValue Zero = CurDAG->getCopyFromReg(Chain,
DL, AArch64::XZR, MVT::i64);
4596 CurDAG->getMachineNode(AArch64::GCSSS1,
DL, MVT::Other, Val, Chain);
4597 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2,
DL, MVT::i64,
4598 MVT::Other, Zero,
SDValue(SS1, 0));
4599 ReplaceNode(
Node, SS2);
4602 case Intrinsic::aarch64_ldaxp:
4603 case Intrinsic::aarch64_ldxp: {
4605 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4610 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4611 MVT::Other, MemAddr, Chain);
4615 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4616 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4617 ReplaceNode(
Node, Ld);
4620 case Intrinsic::aarch64_stlxp:
4621 case Intrinsic::aarch64_stxp: {
4623 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4631 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4633 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4636 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4637 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4639 ReplaceNode(
Node, St);
4642 case Intrinsic::aarch64_neon_ld1x2:
4643 if (VT == MVT::v8i8) {
4644 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4646 }
else if (VT == MVT::v16i8) {
4647 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4649 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4650 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4652 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4653 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4655 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4656 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4658 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4659 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4661 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4662 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4664 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4665 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4669 case Intrinsic::aarch64_neon_ld1x3:
4670 if (VT == MVT::v8i8) {
4671 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4673 }
else if (VT == MVT::v16i8) {
4674 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4676 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4677 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4679 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4680 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4682 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4683 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4685 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4686 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4688 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4689 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4691 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4692 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4696 case Intrinsic::aarch64_neon_ld1x4:
4697 if (VT == MVT::v8i8) {
4698 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4700 }
else if (VT == MVT::v16i8) {
4701 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4703 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4704 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4706 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4707 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4709 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4710 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4712 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4713 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4715 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4716 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4718 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4719 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4723 case Intrinsic::aarch64_neon_ld2:
4724 if (VT == MVT::v8i8) {
4725 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4727 }
else if (VT == MVT::v16i8) {
4728 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4730 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4731 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4733 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4734 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4736 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4737 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4739 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4740 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4742 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4743 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4745 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4746 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4750 case Intrinsic::aarch64_neon_ld3:
4751 if (VT == MVT::v8i8) {
4752 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4754 }
else if (VT == MVT::v16i8) {
4755 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4757 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4758 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4760 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4761 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4763 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4764 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4766 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4767 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4769 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4770 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4772 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4773 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4777 case Intrinsic::aarch64_neon_ld4:
4778 if (VT == MVT::v8i8) {
4779 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4781 }
else if (VT == MVT::v16i8) {
4782 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4784 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4785 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4787 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4788 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4790 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4791 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4793 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4794 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4796 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4797 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4799 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4800 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4804 case Intrinsic::aarch64_neon_ld2r:
4805 if (VT == MVT::v8i8) {
4806 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4808 }
else if (VT == MVT::v16i8) {
4809 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4811 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4812 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4814 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4815 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4817 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4818 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4820 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4821 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4823 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4824 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4826 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4827 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4831 case Intrinsic::aarch64_neon_ld3r:
4832 if (VT == MVT::v8i8) {
4833 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4835 }
else if (VT == MVT::v16i8) {
4836 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4838 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4839 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4841 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4842 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4844 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4845 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4847 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4848 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4850 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4851 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4853 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4854 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4858 case Intrinsic::aarch64_neon_ld4r:
4859 if (VT == MVT::v8i8) {
4860 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4862 }
else if (VT == MVT::v16i8) {
4863 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4865 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4866 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4868 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4869 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
4871 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4872 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
4874 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4875 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
4877 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4878 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
4880 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4881 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
4885 case Intrinsic::aarch64_neon_ld2lane:
4886 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4887 SelectLoadLane(
Node, 2, AArch64::LD2i8);
4889 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4890 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4891 SelectLoadLane(
Node, 2, AArch64::LD2i16);
4893 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4895 SelectLoadLane(
Node, 2, AArch64::LD2i32);
4897 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4899 SelectLoadLane(
Node, 2, AArch64::LD2i64);
4903 case Intrinsic::aarch64_neon_ld3lane:
4904 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4905 SelectLoadLane(
Node, 3, AArch64::LD3i8);
4907 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4908 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4909 SelectLoadLane(
Node, 3, AArch64::LD3i16);
4911 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4913 SelectLoadLane(
Node, 3, AArch64::LD3i32);
4915 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4917 SelectLoadLane(
Node, 3, AArch64::LD3i64);
4921 case Intrinsic::aarch64_neon_ld4lane:
4922 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4923 SelectLoadLane(
Node, 4, AArch64::LD4i8);
4925 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4926 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4927 SelectLoadLane(
Node, 4, AArch64::LD4i16);
4929 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4931 SelectLoadLane(
Node, 4, AArch64::LD4i32);
4933 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4935 SelectLoadLane(
Node, 4, AArch64::LD4i64);
4939 case Intrinsic::aarch64_ld64b:
4940 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
4942 case Intrinsic::aarch64_sve_ld2q_sret: {
4943 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
4946 case Intrinsic::aarch64_sve_ld3q_sret: {
4947 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
4950 case Intrinsic::aarch64_sve_ld4q_sret: {
4951 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
4954 case Intrinsic::aarch64_sve_ld2_sret: {
4955 if (VT == MVT::nxv16i8) {
4956 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
4959 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4960 VT == MVT::nxv8bf16) {
4961 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
4964 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4965 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
4968 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4969 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
4975 case Intrinsic::aarch64_sve_ld1_pn_x2: {
4976 if (VT == MVT::nxv16i8) {
4977 if (Subtarget->hasSME2())
4978 SelectContiguousMultiVectorLoad(
4979 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
4980 else if (Subtarget->hasSVE2p1())
4981 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
4986 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4987 VT == MVT::nxv8bf16) {
4988 if (Subtarget->hasSME2())
4989 SelectContiguousMultiVectorLoad(
4990 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
4991 else if (Subtarget->hasSVE2p1())
4992 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
4997 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4998 if (Subtarget->hasSME2())
4999 SelectContiguousMultiVectorLoad(
5000 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
5001 else if (Subtarget->hasSVE2p1())
5002 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
5007 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5008 if (Subtarget->hasSME2())
5009 SelectContiguousMultiVectorLoad(
5010 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
5011 else if (Subtarget->hasSVE2p1())
5012 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
5020 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5021 if (VT == MVT::nxv16i8) {
5022 if (Subtarget->hasSME2())
5023 SelectContiguousMultiVectorLoad(
5024 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5025 else if (Subtarget->hasSVE2p1())
5026 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
5031 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5032 VT == MVT::nxv8bf16) {
5033 if (Subtarget->hasSME2())
5034 SelectContiguousMultiVectorLoad(
5035 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5036 else if (Subtarget->hasSVE2p1())
5037 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
5042 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5043 if (Subtarget->hasSME2())
5044 SelectContiguousMultiVectorLoad(
5045 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5046 else if (Subtarget->hasSVE2p1())
5047 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
5052 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5053 if (Subtarget->hasSME2())
5054 SelectContiguousMultiVectorLoad(
5055 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5056 else if (Subtarget->hasSVE2p1())
5057 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5065 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5066 if (VT == MVT::nxv16i8) {
5067 if (Subtarget->hasSME2())
5068 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5069 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5070 AArch64::LDNT1B_2Z_PSEUDO);
5071 else if (Subtarget->hasSVE2p1())
5072 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5073 AArch64::LDNT1B_2Z);
5077 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5078 VT == MVT::nxv8bf16) {
5079 if (Subtarget->hasSME2())
5080 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5081 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5082 AArch64::LDNT1H_2Z_PSEUDO);
5083 else if (Subtarget->hasSVE2p1())
5084 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5085 AArch64::LDNT1H_2Z);
5089 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5090 if (Subtarget->hasSME2())
5091 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5092 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5093 AArch64::LDNT1W_2Z_PSEUDO);
5094 else if (Subtarget->hasSVE2p1())
5095 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5096 AArch64::LDNT1W_2Z);
5100 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5101 if (Subtarget->hasSME2())
5102 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5103 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5104 AArch64::LDNT1D_2Z_PSEUDO);
5105 else if (Subtarget->hasSVE2p1())
5106 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5107 AArch64::LDNT1D_2Z);
5114 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5115 if (VT == MVT::nxv16i8) {
5116 if (Subtarget->hasSME2())
5117 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5118 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5119 AArch64::LDNT1B_4Z_PSEUDO);
5120 else if (Subtarget->hasSVE2p1())
5121 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5122 AArch64::LDNT1B_4Z);
5126 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5127 VT == MVT::nxv8bf16) {
5128 if (Subtarget->hasSME2())
5129 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5130 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5131 AArch64::LDNT1H_4Z_PSEUDO);
5132 else if (Subtarget->hasSVE2p1())
5133 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5134 AArch64::LDNT1H_4Z);
5138 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5139 if (Subtarget->hasSME2())
5140 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5141 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5142 AArch64::LDNT1W_4Z_PSEUDO);
5143 else if (Subtarget->hasSVE2p1())
5144 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5145 AArch64::LDNT1W_4Z);
5149 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5150 if (Subtarget->hasSME2())
5151 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5152 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5153 AArch64::LDNT1D_4Z_PSEUDO);
5154 else if (Subtarget->hasSVE2p1())
5155 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5156 AArch64::LDNT1D_4Z);
5163 case Intrinsic::aarch64_sve_ld3_sret: {
5164 if (VT == MVT::nxv16i8) {
5165 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5168 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5169 VT == MVT::nxv8bf16) {
5170 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5173 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5174 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5177 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5178 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5184 case Intrinsic::aarch64_sve_ld4_sret: {
5185 if (VT == MVT::nxv16i8) {
5186 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5189 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5190 VT == MVT::nxv8bf16) {
5191 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5194 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5195 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5198 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5199 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5205 case Intrinsic::aarch64_sme_read_hor_vg2: {
5206 if (VT == MVT::nxv16i8) {
5207 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5208 AArch64::MOVA_2ZMXI_H_B);
5210 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5211 VT == MVT::nxv8bf16) {
5212 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5213 AArch64::MOVA_2ZMXI_H_H);
5215 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5216 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5217 AArch64::MOVA_2ZMXI_H_S);
5219 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5220 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5221 AArch64::MOVA_2ZMXI_H_D);
5226 case Intrinsic::aarch64_sme_read_ver_vg2: {
5227 if (VT == MVT::nxv16i8) {
5228 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5229 AArch64::MOVA_2ZMXI_V_B);
5231 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5232 VT == MVT::nxv8bf16) {
5233 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5234 AArch64::MOVA_2ZMXI_V_H);
5236 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5237 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5238 AArch64::MOVA_2ZMXI_V_S);
5240 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5241 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5242 AArch64::MOVA_2ZMXI_V_D);
5247 case Intrinsic::aarch64_sme_read_hor_vg4: {
5248 if (VT == MVT::nxv16i8) {
5249 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5250 AArch64::MOVA_4ZMXI_H_B);
5252 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5253 VT == MVT::nxv8bf16) {
5254 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5255 AArch64::MOVA_4ZMXI_H_H);
5257 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5258 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5259 AArch64::MOVA_4ZMXI_H_S);
5261 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5262 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5263 AArch64::MOVA_4ZMXI_H_D);
5268 case Intrinsic::aarch64_sme_read_ver_vg4: {
5269 if (VT == MVT::nxv16i8) {
5270 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5271 AArch64::MOVA_4ZMXI_V_B);
5273 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5274 VT == MVT::nxv8bf16) {
5275 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5276 AArch64::MOVA_4ZMXI_V_H);
5278 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5279 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5280 AArch64::MOVA_4ZMXI_V_S);
5282 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5283 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5284 AArch64::MOVA_4ZMXI_V_D);
5289 case Intrinsic::aarch64_sme_read_vg1x2: {
5290 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5291 AArch64::MOVA_VG2_2ZMXI);
5294 case Intrinsic::aarch64_sme_read_vg1x4: {
5295 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5296 AArch64::MOVA_VG4_4ZMXI);
5299 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5300 if (VT == MVT::nxv16i8) {
5301 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5303 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5304 VT == MVT::nxv8bf16) {
5305 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5307 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5308 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5310 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5311 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5316 case Intrinsic::aarch64_sme_readz_vert_x2: {
5317 if (VT == MVT::nxv16i8) {
5318 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
5320 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5321 VT == MVT::nxv8bf16) {
5322 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
5324 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5325 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
5327 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5328 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
5333 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5334 if (VT == MVT::nxv16i8) {
5335 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
5337 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5338 VT == MVT::nxv8bf16) {
5339 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
5341 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5342 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
5344 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5345 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
5350 case Intrinsic::aarch64_sme_readz_vert_x4: {
5351 if (VT == MVT::nxv16i8) {
5352 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
5354 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5355 VT == MVT::nxv8bf16) {
5356 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
5358 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5359 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
5361 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5362 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
5367 case Intrinsic::aarch64_sme_readz_x2: {
5368 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, 7, 1,
5372 case Intrinsic::aarch64_sme_readz_x4: {
5373 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, 7, 1,
5377 case Intrinsic::swift_async_context_addr: {
5380 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5382 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5383 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5384 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5388 CurDAG->RemoveDeadNode(
Node);
5390 auto &MF = CurDAG->getMachineFunction();
5391 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5395 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5396 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5397 Node->getValueType(0),
5398 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5399 AArch64::LUTI2_4ZTZI_S}))
5401 SelectMultiVectorLuti(
Node, 4, Opc, 3);
5404 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5405 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5406 Node->getValueType(0),
5407 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5409 SelectMultiVectorLuti(
Node, 4, Opc, 1);
5412 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5413 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5414 Node->getValueType(0),
5415 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5416 AArch64::LUTI2_2ZTZI_S}))
5418 SelectMultiVectorLuti(
Node, 2, Opc, 7);
5421 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5422 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5423 Node->getValueType(0),
5424 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5425 AArch64::LUTI4_2ZTZI_S}))
5427 SelectMultiVectorLuti(
Node, 2, Opc, 3);
5433 unsigned IntNo =
Node->getConstantOperandVal(0);
5437 case Intrinsic::aarch64_tagp:
5440 case Intrinsic::aarch64_neon_tbl2:
5441 SelectTable(
Node, 2,
5442 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5445 case Intrinsic::aarch64_neon_tbl3:
5446 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5447 : AArch64::TBLv16i8Three,
5450 case Intrinsic::aarch64_neon_tbl4:
5451 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5452 : AArch64::TBLv16i8Four,
5455 case Intrinsic::aarch64_neon_tbx2:
5456 SelectTable(
Node, 2,
5457 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5460 case Intrinsic::aarch64_neon_tbx3:
5461 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5462 : AArch64::TBXv16i8Three,
5465 case Intrinsic::aarch64_neon_tbx4:
5466 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5467 : AArch64::TBXv16i8Four,
5470 case Intrinsic::aarch64_sve_srshl_single_x2:
5471 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5472 Node->getValueType(0),
5473 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5474 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5475 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5477 case Intrinsic::aarch64_sve_srshl_single_x4:
5478 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5479 Node->getValueType(0),
5480 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5481 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5482 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5484 case Intrinsic::aarch64_sve_urshl_single_x2:
5485 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5486 Node->getValueType(0),
5487 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5488 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5489 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5491 case Intrinsic::aarch64_sve_urshl_single_x4:
5492 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5493 Node->getValueType(0),
5494 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5495 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5496 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5498 case Intrinsic::aarch64_sve_srshl_x2:
5499 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5500 Node->getValueType(0),
5501 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5502 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5503 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5505 case Intrinsic::aarch64_sve_srshl_x4:
5506 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5507 Node->getValueType(0),
5508 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5509 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5510 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5512 case Intrinsic::aarch64_sve_urshl_x2:
5513 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5514 Node->getValueType(0),
5515 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5516 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5517 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5519 case Intrinsic::aarch64_sve_urshl_x4:
5520 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5521 Node->getValueType(0),
5522 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5523 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5524 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5526 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5527 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5528 Node->getValueType(0),
5529 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5530 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5531 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5533 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5534 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5535 Node->getValueType(0),
5536 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5537 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5538 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5540 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5541 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5542 Node->getValueType(0),
5543 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5544 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5545 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5547 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5548 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5549 Node->getValueType(0),
5550 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5551 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5552 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5554 case Intrinsic::aarch64_sve_whilege_x2:
5555 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5556 Node->getValueType(0),
5557 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5558 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5559 SelectWhilePair(
Node,
Op);
5561 case Intrinsic::aarch64_sve_whilegt_x2:
5562 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5563 Node->getValueType(0),
5564 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5565 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5566 SelectWhilePair(
Node,
Op);
5568 case Intrinsic::aarch64_sve_whilehi_x2:
5569 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5570 Node->getValueType(0),
5571 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5572 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5573 SelectWhilePair(
Node,
Op);
5575 case Intrinsic::aarch64_sve_whilehs_x2:
5576 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5577 Node->getValueType(0),
5578 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5579 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5580 SelectWhilePair(
Node,
Op);
5582 case Intrinsic::aarch64_sve_whilele_x2:
5583 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5584 Node->getValueType(0),
5585 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5586 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5587 SelectWhilePair(
Node,
Op);
5589 case Intrinsic::aarch64_sve_whilelo_x2:
5590 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5591 Node->getValueType(0),
5592 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5593 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5594 SelectWhilePair(
Node,
Op);
5596 case Intrinsic::aarch64_sve_whilels_x2:
5597 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5598 Node->getValueType(0),
5599 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5600 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5601 SelectWhilePair(
Node,
Op);
5603 case Intrinsic::aarch64_sve_whilelt_x2:
5604 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5605 Node->getValueType(0),
5606 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5607 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5608 SelectWhilePair(
Node,
Op);
5610 case Intrinsic::aarch64_sve_smax_single_x2:
5611 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5612 Node->getValueType(0),
5613 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5614 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5615 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5617 case Intrinsic::aarch64_sve_umax_single_x2:
5618 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5619 Node->getValueType(0),
5620 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5621 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5622 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5624 case Intrinsic::aarch64_sve_fmax_single_x2:
5625 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5626 Node->getValueType(0),
5627 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
5628 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
5629 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5631 case Intrinsic::aarch64_sve_smax_single_x4:
5632 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5633 Node->getValueType(0),
5634 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5635 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5636 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5638 case Intrinsic::aarch64_sve_umax_single_x4:
5639 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5640 Node->getValueType(0),
5641 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5642 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5643 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5645 case Intrinsic::aarch64_sve_fmax_single_x4:
5646 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5647 Node->getValueType(0),
5648 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
5649 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
5650 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5652 case Intrinsic::aarch64_sve_smin_single_x2:
5653 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5654 Node->getValueType(0),
5655 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5656 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5657 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5659 case Intrinsic::aarch64_sve_umin_single_x2:
5660 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5661 Node->getValueType(0),
5662 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5663 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5664 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5666 case Intrinsic::aarch64_sve_fmin_single_x2:
5667 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5668 Node->getValueType(0),
5669 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
5670 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
5671 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5673 case Intrinsic::aarch64_sve_smin_single_x4:
5674 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5675 Node->getValueType(0),
5676 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5677 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5678 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5680 case Intrinsic::aarch64_sve_umin_single_x4:
5681 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5682 Node->getValueType(0),
5683 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5684 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5685 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5687 case Intrinsic::aarch64_sve_fmin_single_x4:
5688 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5689 Node->getValueType(0),
5690 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
5691 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
5692 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5694 case Intrinsic::aarch64_sve_smax_x2:
5695 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5696 Node->getValueType(0),
5697 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5698 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5699 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5701 case Intrinsic::aarch64_sve_umax_x2:
5702 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5703 Node->getValueType(0),
5704 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5705 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5706 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5708 case Intrinsic::aarch64_sve_fmax_x2:
5709 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5710 Node->getValueType(0),
5711 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
5712 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
5713 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5715 case Intrinsic::aarch64_sve_smax_x4:
5716 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5717 Node->getValueType(0),
5718 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5719 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5720 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5722 case Intrinsic::aarch64_sve_umax_x4:
5723 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5724 Node->getValueType(0),
5725 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5726 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5727 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5729 case Intrinsic::aarch64_sve_fmax_x4:
5730 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5731 Node->getValueType(0),
5732 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
5733 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
5734 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5736 case Intrinsic::aarch64_sve_smin_x2:
5737 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5738 Node->getValueType(0),
5739 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5740 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5741 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5743 case Intrinsic::aarch64_sve_umin_x2:
5744 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5745 Node->getValueType(0),
5746 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5747 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5748 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5750 case Intrinsic::aarch64_sve_fmin_x2:
5751 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5752 Node->getValueType(0),
5753 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
5754 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
5755 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5757 case Intrinsic::aarch64_sve_smin_x4:
5758 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5759 Node->getValueType(0),
5760 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5761 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5762 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5764 case Intrinsic::aarch64_sve_umin_x4:
5765 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5766 Node->getValueType(0),
5767 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
5768 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
5769 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5771 case Intrinsic::aarch64_sve_fmin_x4:
5772 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5773 Node->getValueType(0),
5774 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
5775 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
5776 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5778 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
5779 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5780 Node->getValueType(0),
5781 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
5782 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
5783 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5785 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
5786 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5787 Node->getValueType(0),
5788 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
5789 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
5790 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5792 case Intrinsic::aarch64_sve_fminnm_single_x2:
5793 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5794 Node->getValueType(0),
5795 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
5796 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
5797 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5799 case Intrinsic::aarch64_sve_fminnm_single_x4:
5800 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5801 Node->getValueType(0),
5802 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
5803 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
5804 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5806 case Intrinsic::aarch64_sve_fmaxnm_x2:
5807 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5808 Node->getValueType(0),
5809 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
5810 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
5811 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5813 case Intrinsic::aarch64_sve_fmaxnm_x4:
5814 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5815 Node->getValueType(0),
5816 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
5817 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
5818 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5820 case Intrinsic::aarch64_sve_fminnm_x2:
5821 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5822 Node->getValueType(0),
5823 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
5824 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
5825 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5827 case Intrinsic::aarch64_sve_fminnm_x4:
5828 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5829 Node->getValueType(0),
5830 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
5831 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
5832 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5834 case Intrinsic::aarch64_sve_fcvtzs_x2:
5835 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
5837 case Intrinsic::aarch64_sve_scvtf_x2:
5838 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
5840 case Intrinsic::aarch64_sve_fcvtzu_x2:
5841 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
5843 case Intrinsic::aarch64_sve_ucvtf_x2:
5844 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
5846 case Intrinsic::aarch64_sve_fcvtzs_x4:
5847 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
5849 case Intrinsic::aarch64_sve_scvtf_x4:
5850 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
5852 case Intrinsic::aarch64_sve_fcvtzu_x4:
5853 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
5855 case Intrinsic::aarch64_sve_ucvtf_x4:
5856 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
5858 case Intrinsic::aarch64_sve_fcvt_widen_x2:
5859 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVT_2ZZ_H_S);
5861 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
5862 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVTL_2ZZ_H_S);
5864 case Intrinsic::aarch64_sve_sclamp_single_x2:
5865 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5866 Node->getValueType(0),
5867 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
5868 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
5869 SelectClamp(
Node, 2,
Op);
5871 case Intrinsic::aarch64_sve_uclamp_single_x2:
5872 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5873 Node->getValueType(0),
5874 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
5875 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
5876 SelectClamp(
Node, 2,
Op);
5878 case Intrinsic::aarch64_sve_fclamp_single_x2:
5879 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5880 Node->getValueType(0),
5881 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
5882 AArch64::FCLAMP_VG2_2Z2Z_D}))
5883 SelectClamp(
Node, 2,
Op);
5885 case Intrinsic::aarch64_sve_bfclamp_single_x2:
5886 SelectClamp(
Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
5888 case Intrinsic::aarch64_sve_sclamp_single_x4:
5889 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5890 Node->getValueType(0),
5891 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
5892 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
5893 SelectClamp(
Node, 4,
Op);
5895 case Intrinsic::aarch64_sve_uclamp_single_x4:
5896 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5897 Node->getValueType(0),
5898 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
5899 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
5900 SelectClamp(
Node, 4,
Op);
5902 case Intrinsic::aarch64_sve_fclamp_single_x4:
5903 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5904 Node->getValueType(0),
5905 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
5906 AArch64::FCLAMP_VG4_4Z4Z_D}))
5907 SelectClamp(
Node, 4,
Op);
5909 case Intrinsic::aarch64_sve_bfclamp_single_x4:
5910 SelectClamp(
Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
5912 case Intrinsic::aarch64_sve_add_single_x2:
5913 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5914 Node->getValueType(0),
5915 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
5916 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
5917 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5919 case Intrinsic::aarch64_sve_add_single_x4:
5920 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5921 Node->getValueType(0),
5922 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
5923 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
5924 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5926 case Intrinsic::aarch64_sve_zip_x2:
5927 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5928 Node->getValueType(0),
5929 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
5930 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
5931 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5933 case Intrinsic::aarch64_sve_zipq_x2:
5934 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5935 AArch64::ZIP_VG2_2ZZZ_Q);
5937 case Intrinsic::aarch64_sve_zip_x4:
5938 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5939 Node->getValueType(0),
5940 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
5941 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
5942 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5944 case Intrinsic::aarch64_sve_zipq_x4:
5945 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5946 AArch64::ZIP_VG4_4Z4Z_Q);
5948 case Intrinsic::aarch64_sve_uzp_x2:
5949 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5950 Node->getValueType(0),
5951 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
5952 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
5953 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5955 case Intrinsic::aarch64_sve_uzpq_x2:
5956 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5957 AArch64::UZP_VG2_2ZZZ_Q);
5959 case Intrinsic::aarch64_sve_uzp_x4:
5960 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5961 Node->getValueType(0),
5962 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
5963 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
5964 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5966 case Intrinsic::aarch64_sve_uzpq_x4:
5967 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5968 AArch64::UZP_VG4_4Z4Z_Q);
5970 case Intrinsic::aarch64_sve_sel_x2:
5971 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5972 Node->getValueType(0),
5973 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
5974 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
5975 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
5977 case Intrinsic::aarch64_sve_sel_x4:
5978 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5979 Node->getValueType(0),
5980 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
5981 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
5982 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
5984 case Intrinsic::aarch64_sve_frinta_x2:
5985 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
5987 case Intrinsic::aarch64_sve_frinta_x4:
5988 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
5990 case Intrinsic::aarch64_sve_frintm_x2:
5991 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
5993 case Intrinsic::aarch64_sve_frintm_x4:
5994 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
5996 case Intrinsic::aarch64_sve_frintn_x2:
5997 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
5999 case Intrinsic::aarch64_sve_frintn_x4:
6000 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
6002 case Intrinsic::aarch64_sve_frintp_x2:
6003 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
6005 case Intrinsic::aarch64_sve_frintp_x4:
6006 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
6008 case Intrinsic::aarch64_sve_sunpk_x2:
6009 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6010 Node->getValueType(0),
6011 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6012 AArch64::SUNPK_VG2_2ZZ_D}))
6013 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6015 case Intrinsic::aarch64_sve_uunpk_x2:
6016 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6017 Node->getValueType(0),
6018 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6019 AArch64::UUNPK_VG2_2ZZ_D}))
6020 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6022 case Intrinsic::aarch64_sve_sunpk_x4:
6023 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6024 Node->getValueType(0),
6025 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6026 AArch64::SUNPK_VG4_4Z2Z_D}))
6027 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6029 case Intrinsic::aarch64_sve_uunpk_x4:
6030 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6031 Node->getValueType(0),
6032 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6033 AArch64::UUNPK_VG4_4Z2Z_D}))
6034 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6036 case Intrinsic::aarch64_sve_pext_x2: {
6037 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6038 Node->getValueType(0),
6039 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6040 AArch64::PEXT_2PCI_D}))
6041 SelectPExtPair(
Node,
Op);
6048 unsigned IntNo =
Node->getConstantOperandVal(1);
6049 if (
Node->getNumOperands() >= 3)
6050 VT =
Node->getOperand(2)->getValueType(0);
6054 case Intrinsic::aarch64_neon_st1x2: {
6055 if (VT == MVT::v8i8) {
6056 SelectStore(
Node, 2, AArch64::ST1Twov8b);
6058 }
else if (VT == MVT::v16i8) {
6059 SelectStore(
Node, 2, AArch64::ST1Twov16b);
6061 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6062 VT == MVT::v4bf16) {
6063 SelectStore(
Node, 2, AArch64::ST1Twov4h);
6065 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6066 VT == MVT::v8bf16) {
6067 SelectStore(
Node, 2, AArch64::ST1Twov8h);
6069 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6070 SelectStore(
Node, 2, AArch64::ST1Twov2s);
6072 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6073 SelectStore(
Node, 2, AArch64::ST1Twov4s);
6075 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6076 SelectStore(
Node, 2, AArch64::ST1Twov2d);
6078 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6079 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6084 case Intrinsic::aarch64_neon_st1x3: {
6085 if (VT == MVT::v8i8) {
6086 SelectStore(
Node, 3, AArch64::ST1Threev8b);
6088 }
else if (VT == MVT::v16i8) {
6089 SelectStore(
Node, 3, AArch64::ST1Threev16b);
6091 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6092 VT == MVT::v4bf16) {
6093 SelectStore(
Node, 3, AArch64::ST1Threev4h);
6095 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6096 VT == MVT::v8bf16) {
6097 SelectStore(
Node, 3, AArch64::ST1Threev8h);
6099 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6100 SelectStore(
Node, 3, AArch64::ST1Threev2s);
6102 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6103 SelectStore(
Node, 3, AArch64::ST1Threev4s);
6105 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6106 SelectStore(
Node, 3, AArch64::ST1Threev2d);
6108 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6109 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6114 case Intrinsic::aarch64_neon_st1x4: {
6115 if (VT == MVT::v8i8) {
6116 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
6118 }
else if (VT == MVT::v16i8) {
6119 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
6121 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6122 VT == MVT::v4bf16) {
6123 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
6125 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6126 VT == MVT::v8bf16) {
6127 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
6129 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6130 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
6132 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6133 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
6135 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6136 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
6138 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6139 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6144 case Intrinsic::aarch64_neon_st2: {
6145 if (VT == MVT::v8i8) {
6146 SelectStore(
Node, 2, AArch64::ST2Twov8b);
6148 }
else if (VT == MVT::v16i8) {
6149 SelectStore(
Node, 2, AArch64::ST2Twov16b);
6151 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6152 VT == MVT::v4bf16) {
6153 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6155 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6156 VT == MVT::v8bf16) {
6157 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6159 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6160 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6162 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6163 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6165 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6166 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6168 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6169 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6174 case Intrinsic::aarch64_neon_st3: {
6175 if (VT == MVT::v8i8) {
6176 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6178 }
else if (VT == MVT::v16i8) {
6179 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6181 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6182 VT == MVT::v4bf16) {
6183 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6185 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6186 VT == MVT::v8bf16) {
6187 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6189 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6190 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6192 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6193 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6195 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6196 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6198 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6199 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6204 case Intrinsic::aarch64_neon_st4: {
6205 if (VT == MVT::v8i8) {
6206 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6208 }
else if (VT == MVT::v16i8) {
6209 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6211 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6212 VT == MVT::v4bf16) {
6213 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6215 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6216 VT == MVT::v8bf16) {
6217 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6219 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6220 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6222 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6223 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6225 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6226 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6228 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6229 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6234 case Intrinsic::aarch64_neon_st2lane: {
6235 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6236 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6238 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6239 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6240 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6242 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6244 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6246 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6248 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6253 case Intrinsic::aarch64_neon_st3lane: {
6254 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6255 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6257 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6258 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6259 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6261 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6263 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6265 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6267 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6272 case Intrinsic::aarch64_neon_st4lane: {
6273 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6274 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6276 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6277 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6278 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6280 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6282 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6284 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6286 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6291 case Intrinsic::aarch64_sve_st2q: {
6292 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6295 case Intrinsic::aarch64_sve_st3q: {
6296 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6299 case Intrinsic::aarch64_sve_st4q: {
6300 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6303 case Intrinsic::aarch64_sve_st2: {
6304 if (VT == MVT::nxv16i8) {
6305 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6307 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6308 VT == MVT::nxv8bf16) {
6309 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6311 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6312 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6314 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6315 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6320 case Intrinsic::aarch64_sve_st3: {
6321 if (VT == MVT::nxv16i8) {
6322 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6324 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6325 VT == MVT::nxv8bf16) {
6326 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6328 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6329 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6331 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6332 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6337 case Intrinsic::aarch64_sve_st4: {
6338 if (VT == MVT::nxv16i8) {
6339 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6341 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6342 VT == MVT::nxv8bf16) {
6343 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6345 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6346 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6348 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6349 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6358 if (VT == MVT::v8i8) {
6359 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6361 }
else if (VT == MVT::v16i8) {
6362 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6364 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6365 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6367 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6368 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6370 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6371 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6373 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6374 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6376 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6377 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6379 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6380 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6386 if (VT == MVT::v8i8) {
6387 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6389 }
else if (VT == MVT::v16i8) {
6390 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6392 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6393 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6395 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6396 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6398 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6399 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6401 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6402 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6404 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6405 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6407 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6408 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6414 if (VT == MVT::v8i8) {
6415 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6417 }
else if (VT == MVT::v16i8) {
6418 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6420 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6421 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6423 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6424 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6426 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6427 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6429 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6430 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6432 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6433 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6435 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6436 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6442 if (VT == MVT::v8i8) {
6443 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6445 }
else if (VT == MVT::v16i8) {
6446 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6448 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6449 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6451 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6452 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6454 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6455 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6457 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6458 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6460 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6461 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6463 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6464 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6470 if (VT == MVT::v8i8) {
6471 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6473 }
else if (VT == MVT::v16i8) {
6474 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6476 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6477 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6479 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6480 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6482 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6483 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6485 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6486 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6488 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6489 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6491 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6492 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6498 if (VT == MVT::v8i8) {
6499 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6501 }
else if (VT == MVT::v16i8) {
6502 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6504 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6505 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6507 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6508 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6510 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6511 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6513 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6514 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6516 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6517 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6519 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6520 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6526 if (VT == MVT::v8i8) {
6527 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6529 }
else if (VT == MVT::v16i8) {
6530 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6532 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6533 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6535 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6536 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6538 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6539 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6541 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6542 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6544 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6545 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6547 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6548 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6554 if (VT == MVT::v8i8) {
6555 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6557 }
else if (VT == MVT::v16i8) {
6558 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6560 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6561 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6563 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6564 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6566 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6567 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6569 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6570 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6572 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6573 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6575 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6576 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6582 if (VT == MVT::v8i8) {
6583 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6585 }
else if (VT == MVT::v16i8) {
6586 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6588 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6589 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6591 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6592 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6594 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6595 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6597 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6598 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6600 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6601 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6603 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6604 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6610 if (VT == MVT::v8i8) {
6611 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6613 }
else if (VT == MVT::v16i8) {
6614 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6616 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6617 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6619 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6620 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6622 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6623 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6625 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6626 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6628 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6629 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6631 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6632 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6638 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6639 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6641 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6642 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6643 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6645 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6647 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6649 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6651 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6657 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6658 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6660 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6661 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6662 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6664 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6666 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6668 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6670 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6676 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6677 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6679 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6680 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6681 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6683 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6685 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6687 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6689 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6695 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6696 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6698 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6699 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6700 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6702 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6704 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6706 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6708 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6714 VT =
Node->getOperand(1).getValueType();
6715 if (VT == MVT::v8i8) {
6716 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6718 }
else if (VT == MVT::v16i8) {
6719 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6721 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6722 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6724 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6725 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6727 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6728 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6730 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6731 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6733 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6734 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6736 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6737 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6743 VT =
Node->getOperand(1).getValueType();
6744 if (VT == MVT::v8i8) {
6745 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6747 }
else if (VT == MVT::v16i8) {
6748 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6750 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6751 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6753 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6754 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6756 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6757 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6759 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6760 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6762 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6763 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
6765 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6766 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6772 VT =
Node->getOperand(1).getValueType();
6773 if (VT == MVT::v8i8) {
6774 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
6776 }
else if (VT == MVT::v16i8) {
6777 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
6779 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6780 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
6782 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6783 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
6785 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6786 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
6788 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6789 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
6791 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6792 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
6794 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6795 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6801 VT =
Node->getOperand(1).getValueType();
6802 if (VT == MVT::v8i8) {
6803 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
6805 }
else if (VT == MVT::v16i8) {
6806 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
6808 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6809 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
6811 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6812 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
6814 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6815 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
6817 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6818 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
6820 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6821 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6823 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6824 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
6830 VT =
Node->getOperand(1).getValueType();
6831 if (VT == MVT::v8i8) {
6832 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
6834 }
else if (VT == MVT::v16i8) {
6835 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
6837 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6838 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
6840 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
6841 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
6843 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6844 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
6846 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6847 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
6849 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6850 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6852 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6853 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
6859 VT =
Node->getOperand(1).getValueType();
6860 if (VT == MVT::v8i8) {
6861 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
6863 }
else if (VT == MVT::v16i8) {
6864 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
6866 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6867 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
6869 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6870 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
6872 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6873 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
6875 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6876 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
6878 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6879 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6881 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6882 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
6888 VT =
Node->getOperand(1).getValueType();
6889 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6890 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
6892 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6893 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6894 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
6896 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6898 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
6900 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6902 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
6908 VT =
Node->getOperand(1).getValueType();
6909 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6910 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
6912 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6913 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6914 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
6916 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6918 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
6920 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6922 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
6928 VT =
Node->getOperand(1).getValueType();
6929 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6930 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
6932 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6933 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6934 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
6936 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6938 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
6940 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6942 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
6948 if (VT == MVT::nxv16i8) {
6949 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
6951 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6952 VT == MVT::nxv8bf16) {
6953 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
6955 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6956 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
6958 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6959 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
6965 if (VT == MVT::nxv16i8) {
6966 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
6968 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6969 VT == MVT::nxv8bf16) {
6970 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
6972 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6973 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
6975 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6976 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
6982 if (VT == MVT::nxv16i8) {
6983 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
6985 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6986 VT == MVT::nxv8bf16) {
6987 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
6989 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6990 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
6992 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6993 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
7008 return new AArch64DAGToDAGISelLegacy(
TM, OptLevel);
7020 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
7024 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7025 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7039 if (isa<MemSDNode>(Root))
7040 return cast<MemSDNode>(Root)->getMemoryVT();
7042 if (isa<MemIntrinsicSDNode>(Root))
7043 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
7045 const unsigned Opcode = Root->
getOpcode();
7053 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
7055 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
7075 case Intrinsic::aarch64_sme_ldr:
7076 case Intrinsic::aarch64_sme_str:
7077 return MVT::nxv16i8;
7078 case Intrinsic::aarch64_sve_prf:
7083 case Intrinsic::aarch64_sve_ld2_sret:
7084 case Intrinsic::aarch64_sve_ld2q_sret:
7087 case Intrinsic::aarch64_sve_st2q:
7090 case Intrinsic::aarch64_sve_ld3_sret:
7091 case Intrinsic::aarch64_sve_ld3q_sret:
7094 case Intrinsic::aarch64_sve_st3q:
7097 case Intrinsic::aarch64_sve_ld4_sret:
7098 case Intrinsic::aarch64_sve_ld4q_sret:
7101 case Intrinsic::aarch64_sve_st4q:
7104 case Intrinsic::aarch64_sve_ld1udq:
7105 case Intrinsic::aarch64_sve_st1dq:
7106 return EVT(MVT::nxv1i64);
7107 case Intrinsic::aarch64_sve_ld1uwq:
7108 case Intrinsic::aarch64_sve_st1wq:
7109 return EVT(MVT::nxv1i32);
7116template <
int64_t Min,
int64_t Max>
7117bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
7125 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
7130 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
7149 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
7151 if ((MulImm % MemWidthBytes) != 0)
7154 int64_t
Offset = MulImm / MemWidthBytes;
7155 if (Offset < Min || Offset > Max)
7158 Base =
N.getOperand(0);
7160 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7167 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7173bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7191 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7192 int64_t ImmOff =
C->getSExtValue();
7193 unsigned Size = 1 << Scale;
7202 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7204 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7214 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7215 if (
C->getZExtValue() == Scale) {
7224bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7231bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7232 EVT VT =
N.getValueType();
7236bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7241 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7242 int64_t ImmOff =
C->getSExtValue();
7243 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7244 Base =
N.getOperand(0);
7245 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7252 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
unsigned const TargetRegisterInfo * TRI
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
static constexpr unsigned MaxRecursionDepth
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Not(const Pred &P) -> Not< Pred >
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.