22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
49 AArch64DAGToDAGISel() =
delete;
66 std::vector<SDValue> &OutOps)
override;
68 template <
signed Low,
signed High,
signed Scale>
76 return SelectShiftedRegister(
N,
false, Reg, Shift);
79 return SelectShiftedRegister(
N,
true, Reg, Shift);
82 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
85 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
88 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
91 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
94 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
97 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
100 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
103 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
106 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
109 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
112 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
115 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
118 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
121 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
124 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
127 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
130 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
132 template <
unsigned Size,
unsigned Max>
136 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
138 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
139 int64_t
C = CI->getSExtValue();
147 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
154 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
160 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
165 N =
N->getOperand(0);
167 !isa<ConstantSDNode>(
N->getOperand(1)))
169 EVT VT =
N->getValueType(0);
170 EVT LVT =
N->getOperand(0).getValueType();
171 unsigned Index =
N->getConstantOperandVal(1);
175 Res =
N->getOperand(0);
183 EVT VT =
Op.getValueType();
184 unsigned ShtAmt =
N->getConstantOperandVal(1);
191 Op.getOperand(1).getConstantOperandVal(0)
192 <<
Op.getOperand(1).getConstantOperandVal(1));
194 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
196 Op.getOperand(1).getConstantOperandVal(0));
200 if (Imm != 1ULL << (ShtAmt - 1))
203 Res1 =
Op.getOperand(0);
204 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
208 bool SelectDupZeroOrUndef(
SDValue N) {
209 switch(
N->getOpcode()) {
214 auto Opnd0 =
N->getOperand(0);
229 switch(
N->getOpcode()) {
232 auto Opnd0 =
N->getOperand(0);
244 bool SelectDupNegativeZero(
SDValue N) {
245 switch(
N->getOpcode()) {
249 return Const && Const->isZero() && Const->isNegative();
256 template<MVT::SimpleValueType VT>
258 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
261 template <MVT::SimpleValueType VT,
bool Negate>
263 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
266 template <MVT::SimpleValueType VT>
268 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
271 template <MVT::SimpleValueType VT,
bool Invert = false>
273 return SelectSVELogicalImm(
N, VT, Imm, Invert);
276 template <MVT::SimpleValueType VT>
278 return SelectSVEArithImm(
N, VT, Imm);
281 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
283 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
290 EVT EltVT =
N->getValueType(0).getVectorElementType();
291 return SelectSVEShiftImm(
N->getOperand(0), 1,
297 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
299 if (!isa<ConstantSDNode>(
N))
302 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
304 MulImm = 1LL << MulImm;
306 if ((MulImm % std::abs(Scale)) != 0)
310 if ((MulImm >= Min) && (MulImm <= Max)) {
311 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
318 template <
signed Max,
signed Scale>
320 if (!isa<ConstantSDNode>(
N))
323 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
325 if (MulImm >= 0 && MulImm <= Max) {
327 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
334 template <
unsigned BaseReg,
unsigned Max>
336 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
342 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
365 const unsigned SubRegs[]);
367 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
369 bool tryIndexedLoad(
SDNode *
N);
371 bool trySelectStackSlotTagP(
SDNode *
N);
374 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
376 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
378 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
379 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
380 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
381 unsigned Opc_rr,
unsigned Opc_ri,
382 bool IsIntr =
false);
383 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
384 unsigned Scale,
unsigned Opc_ri,
386 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
387 bool IsZmMulti,
unsigned Opcode,
388 bool HasPred =
false);
389 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
390 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
391 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
392 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
393 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
394 bool IsTupleInput,
unsigned Opc);
395 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
397 template <
unsigned MaxIdx,
unsigned Scale>
398 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
403 template <
int64_t Min,
int64_t Max>
407 template <
unsigned Scale>
409 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
412 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
415 template <
unsigned MaxIdx,
unsigned Scale>
420 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
421 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
422 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
423 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
424 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
425 unsigned Opc_rr,
unsigned Opc_ri);
426 std::tuple<unsigned, SDValue, SDValue>
427 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
431 bool tryBitfieldExtractOp(
SDNode *
N);
432 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
433 bool tryBitfieldInsertOp(
SDNode *
N);
434 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
435 bool tryShiftAmountMod(
SDNode *
N);
437 bool tryReadRegister(
SDNode *
N);
438 bool tryWriteRegister(
SDNode *
N);
440 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
441 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
446#include "AArch64GenDAGISel.inc"
454 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
456 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
469 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
470 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
471 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
474 template<
unsigned RegW
idth>
476 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
479 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
481 template<
unsigned RegW
idth>
483 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
489 bool SelectCMP_SWAP(
SDNode *
N);
499 bool AllowSaturation,
SDValue &Imm);
507 bool SelectAllActivePredicate(
SDValue N);
512char AArch64DAGToDAGISel::ID = 0;
520 Imm =
C->getZExtValue();
537 return N->getOpcode() == Opc &&
548 return Imm == ImmExpected;
552bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
554 std::vector<SDValue> &OutOps) {
555 switch(ConstraintID) {
558 case InlineAsm::ConstraintCode::m:
559 case InlineAsm::ConstraintCode::o:
560 case InlineAsm::ConstraintCode::Q:
566 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
568 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
569 dl,
Op.getValueType(),
571 OutOps.push_back(NewOp);
587 if (!isa<ConstantSDNode>(
N.getNode()))
590 uint64_t Immed =
N.getNode()->getAsZExtVal();
593 if (Immed >> 12 == 0) {
595 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
603 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
604 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
617 if (!isa<ConstantSDNode>(
N.getNode()))
621 uint64_t Immed =
N.getNode()->getAsZExtVal();
629 if (
N.getValueType() == MVT::i32)
632 Immed = ~Immed + 1ULL;
633 if (Immed & 0xFFFFFFFFFF000000ULL)
636 Immed &= 0xFFFFFFULL;
637 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
644 switch (
N.getOpcode()) {
663 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
666 unsigned ShiftVal = CSD->getZExtValue();
675 if (!isa<MemSDNode>(*UI))
677 if (!isa<MemSDNode>(*UII))
684bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
687 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
692 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
714bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
716 EVT VT =
N.getValueType();
717 if (VT != MVT::i32 && VT != MVT::i64)
720 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
726 unsigned LHSOpcode =
LHS->getOpcode();
740 unsigned LowZBits, MaskLen;
744 unsigned BitWidth =
N.getValueSizeInBits();
751 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
754 NewShiftC = LowZBits - ShiftAmtC;
755 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
761 NewShiftC = LowZBits + ShiftAmtC;
774 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
776 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
780 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
782 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
783 NewShiftAmt, BitWidthMinus1),
786 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
798 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
800 SrcVT =
N.getOperand(0).getValueType();
802 if (!IsLoadStore && SrcVT == MVT::i8)
804 else if (!IsLoadStore && SrcVT == MVT::i16)
806 else if (SrcVT == MVT::i32)
808 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
813 EVT SrcVT =
N.getOperand(0).getValueType();
814 if (!IsLoadStore && SrcVT == MVT::i8)
816 else if (!IsLoadStore && SrcVT == MVT::i16)
818 else if (SrcVT == MVT::i32)
820 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
848bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
851 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
856 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
857 V.getConstantOperandVal(1) <= 4 &&
870bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
872 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
882 unsigned BitSize =
N.getValueSizeInBits();
883 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
886 Reg =
N.getOperand(0);
887 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
888 return isWorthFoldingALU(
N,
true);
899 if (
N.getValueType() == MVT::i32)
907template<
signed Low,
signed High,
signed Scale>
909 if (!isa<ConstantSDNode>(
N))
912 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
913 if ((MulImm % std::abs(Scale)) == 0) {
914 int64_t RDVLImm = MulImm / Scale;
915 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
916 Imm = CurDAG->getTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
926bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
928 unsigned ShiftVal = 0;
943 Reg =
N.getOperand(0).getOperand(0);
949 Reg =
N.getOperand(0);
954 unsigned Opc =
N.getOpcode();
955 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
972 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
974 return isWorthFoldingALU(
N);
979bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
981 unsigned ShiftVal = 0;
995 Reg =
N.getOperand(0);
996 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
998 return isWorthFoldingALU(
N);
1007 for (
auto *
Use :
N->uses()) {
1034bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1035 unsigned BW,
unsigned Size,
1042 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1044 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1050 if (CurDAG->isBaseWithConstantOffset(
N)) {
1051 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1053 int64_t RHSC =
RHS->getSExtValue();
1055 int64_t
Range = 0x1LL << (BW - 1);
1057 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1058 RHSC < (Range << Scale)) {
1059 Base =
N.getOperand(0);
1061 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1064 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1073 if ((RHSC & (
Size - 1)) == 0 && RHSC < (Range << Scale)) {
1074 Base =
N.getOperand(0);
1076 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1079 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1090 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1097bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1103 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1105 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1111 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1112 Base =
N.getOperand(0);
1113 OffImm =
N.getOperand(1);
1122 if (CurDAG->isBaseWithConstantOffset(
N)) {
1123 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1124 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1127 Base =
N.getOperand(0);
1129 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1132 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1140 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1148 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1157bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1160 if (!CurDAG->isBaseWithConstantOffset(
N))
1162 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1163 int64_t RHSC =
RHS->getSExtValue();
1164 if (RHSC >= -256 && RHSC < 256) {
1165 Base =
N.getOperand(0);
1167 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1169 Base = CurDAG->getTargetFrameIndex(
1172 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1182 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1189bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1209 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1215 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1218 return isWorthFoldingAddr(
N,
Size);
1221bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1233 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1241 if (!isa<MemSDNode>(*UI))
1246 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1249 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1250 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1252 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1257 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1258 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1260 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1265 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1269 if (IsExtendedRegisterWorthFolding &&
1276 if (isWorthFoldingAddr(LHS,
Size))
1281 if (IsExtendedRegisterWorthFolding &&
1288 if (isWorthFoldingAddr(RHS,
Size))
1300 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1303 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1305 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1306 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1310bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1325 if (!isa<MemSDNode>(*UI))
1340 if (isa<ConstantSDNode>(RHS)) {
1341 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1351 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1354 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1358 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1361 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1362 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1364 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1369 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1370 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1372 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1379 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1380 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1386 static const unsigned RegClassIDs[] = {
1387 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1388 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1389 AArch64::dsub2, AArch64::dsub3};
1395 static const unsigned RegClassIDs[] = {
1396 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1397 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1398 AArch64::qsub2, AArch64::qsub3};
1404 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1405 AArch64::ZPR3RegClassID,
1406 AArch64::ZPR4RegClassID};
1407 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1408 AArch64::zsub2, AArch64::zsub3};
1418 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1419 AArch64::ZPR4Mul4RegClassID};
1420 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1421 AArch64::zsub2, AArch64::zsub3};
1426 const unsigned RegClassIDs[],
1427 const unsigned SubRegs[]) {
1430 if (Regs.
size() == 1)
1441 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1444 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1446 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1450 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1454void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1457 EVT VT =
N->getValueType(0);
1459 unsigned ExtOff = isExt;
1462 unsigned Vec0Off = ExtOff + 1;
1464 N->op_begin() + Vec0Off + NumVecs);
1471 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1472 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1475bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1477 if (
LD->isUnindexed())
1479 EVT VT =
LD->getMemoryVT();
1480 EVT DstVT =
N->getValueType(0);
1487 unsigned Opcode = 0;
1490 bool InsertTo64 =
false;
1492 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1493 else if (VT == MVT::i32) {
1495 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1497 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1499 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1505 }
else if (VT == MVT::i16) {
1507 if (DstVT == MVT::i64)
1508 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1510 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1512 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1513 InsertTo64 = DstVT == MVT::i64;
1518 }
else if (VT == MVT::i8) {
1520 if (DstVT == MVT::i64)
1521 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1523 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1525 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1526 InsertTo64 = DstVT == MVT::i64;
1531 }
else if (VT == MVT::f16) {
1532 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1533 }
else if (VT == MVT::bf16) {
1534 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1535 }
else if (VT == MVT::f32) {
1536 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1538 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1540 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1548 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1550 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1555 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1560 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1562 SDValue(CurDAG->getMachineNode(
1563 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1564 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1569 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1572 CurDAG->RemoveDeadNode(
N);
1576void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1577 unsigned SubRegIdx) {
1579 EVT VT =
N->getValueType(0);
1585 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1587 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1589 for (
unsigned i = 0; i < NumVecs; ++i)
1591 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1597 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1599 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1602 CurDAG->RemoveDeadNode(
N);
1605void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1606 unsigned Opc,
unsigned SubRegIdx) {
1608 EVT VT =
N->getValueType(0);
1615 const EVT ResTys[] = {MVT::i64,
1616 MVT::Untyped, MVT::Other};
1618 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1626 ReplaceUses(
SDValue(
N, 0), SuperReg);
1628 for (
unsigned i = 0; i < NumVecs; ++i)
1630 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1634 CurDAG->RemoveDeadNode(
N);
1640std::tuple<unsigned, SDValue, SDValue>
1641AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1647 SDValue NewOffset = OldOffset;
1649 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1650 N, OldBase, NewBase, NewOffset);
1654 const bool IsRegReg =
1655 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1658 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1671template <SelectTypeKind Kind>
1682 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1687 if (EltVT != MVT::i1)
1691 if (EltVT != MVT::f16 && EltVT != MVT::f32 && EltVT != MVT::f64)
1720void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1723 if (
Imm->getZExtValue() > 1)
1727 EVT VT =
N->getValueType(0);
1728 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1729 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1732 for (
unsigned I = 0;
I < 2; ++
I)
1733 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1734 AArch64::psub0 +
I,
DL, VT, SuperReg));
1736 CurDAG->RemoveDeadNode(
N);
1739void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1741 EVT VT =
N->getValueType(0);
1743 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1745 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1748 for (
unsigned I = 0;
I < 2; ++
I)
1749 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1750 AArch64::psub0 +
I,
DL, VT, SuperReg));
1752 CurDAG->RemoveDeadNode(
N);
1755void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1757 EVT VT =
N->getValueType(0);
1759 SDValue Ops = createZTuple(Regs);
1763 for (
unsigned i = 0; i < NumVecs; ++i)
1764 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1765 AArch64::zsub0 + i,
DL, VT, SuperReg));
1767 CurDAG->RemoveDeadNode(
N);
1770void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1775 assert(Opcode != 0 &&
"Unexpected opcode");
1778 EVT VT =
N->getValueType(0);
1779 unsigned FirstVecIdx = HasPred ? 2 : 1;
1781 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1783 N->op_begin() + StartIdx + NumVecs);
1784 return createZMulTuple(Regs);
1787 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1791 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1793 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1797 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1798 N->getOperand(1), Zdn, Zm);
1800 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1802 for (
unsigned i = 0; i < NumVecs; ++i)
1803 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1804 AArch64::zsub0 + i,
DL, VT, SuperReg));
1806 CurDAG->RemoveDeadNode(
N);
1809void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1810 unsigned Scale,
unsigned Opc_ri,
1811 unsigned Opc_rr,
bool IsIntr) {
1812 assert(Scale < 5 &&
"Invalid scaling value.");
1814 EVT VT =
N->getValueType(0);
1820 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1821 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1822 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1824 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1828 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1830 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1832 for (
unsigned i = 0; i < NumVecs; ++i)
1833 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1834 AArch64::zsub0 + i,
DL, VT, SuperReg));
1837 unsigned ChainIdx = NumVecs;
1839 CurDAG->RemoveDeadNode(
N);
1842void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1847 assert(Scale < 4 &&
"Invalid scaling value.");
1849 EVT VT =
N->getValueType(0);
1857 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1863 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1865 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1867 for (
unsigned i = 0; i < NumVecs; ++i)
1868 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1869 AArch64::zsub0 + i,
DL, VT, SuperReg));
1872 unsigned ChainIdx = NumVecs;
1874 CurDAG->RemoveDeadNode(
N);
1877void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1879 if (
N->getValueType(0) != MVT::nxv4f32)
1881 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1884void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
1885 unsigned NumOutVecs,
1888 if (
Imm->getZExtValue() > MaxImm)
1892 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
1896 EVT VT =
Node->getValueType(0);
1899 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
1902 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
1903 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
1904 AArch64::zsub0 +
I,
DL, VT, SuperReg));
1907 unsigned ChainIdx = NumOutVecs;
1909 CurDAG->RemoveDeadNode(
Node);
1912void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
1915 EVT VT =
N->getValueType(0);
1918 SDValue Zd = createZMulTuple(Regs);
1919 SDValue Zn =
N->getOperand(1 + NumVecs);
1920 SDValue Zm =
N->getOperand(2 + NumVecs);
1926 for (
unsigned i = 0; i < NumVecs; ++i)
1927 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1928 AArch64::zsub0 + i,
DL, VT, SuperReg));
1930 CurDAG->RemoveDeadNode(
N);
1960template <
unsigned MaxIdx,
unsigned Scale>
1961void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
1962 unsigned BaseReg,
unsigned Op) {
1963 unsigned TileNum = 0;
1964 if (BaseReg != AArch64::ZA)
1965 TileNum =
N->getConstantOperandVal(2);
1971 if (BaseReg == AArch64::ZA)
1972 SliceBase =
N->getOperand(2);
1974 SliceBase =
N->getOperand(3);
1976 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
1982 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
1984 EVT VT =
N->getValueType(0);
1985 for (
unsigned I = 0;
I < NumVecs; ++
I)
1987 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
1990 unsigned ChainIdx = NumVecs;
1992 CurDAG->RemoveDeadNode(
N);
1995void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
1996 unsigned NumOutVecs,
2000 EVT VT =
N->getValueType(0);
2001 unsigned NumInVecs =
N->getNumOperands() - 1;
2005 assert((NumInVecs == 2 || NumInVecs == 4) &&
2006 "Don't know how to handle multi-register input!");
2008 N->op_begin() + 1 + NumInVecs);
2012 for (
unsigned I = 0;
I < NumInVecs;
I++)
2016 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2019 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2020 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2021 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2022 CurDAG->RemoveDeadNode(
N);
2025void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2028 EVT VT =
N->getOperand(2)->getValueType(0);
2035 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2036 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2040 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2045void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2046 unsigned Scale,
unsigned Opc_rr,
2052 SDValue RegSeq = createZTuple(Regs);
2057 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2058 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2059 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2061 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2065 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2077 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2078 int FI = FINode->getIndex();
2080 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2087void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2090 EVT VT =
N->getOperand(2)->getValueType(0);
2091 const EVT ResTys[] = {MVT::i64,
2100 N->getOperand(NumVecs + 1),
2101 N->getOperand(NumVecs + 2),
2103 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2143void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2146 EVT VT =
N->getValueType(0);
2154 WidenVector(*CurDAG));
2158 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2160 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2162 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2163 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2164 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2168 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2169 AArch64::qsub2, AArch64::qsub3 };
2170 for (
unsigned i = 0; i < NumVecs; ++i) {
2171 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2178 CurDAG->RemoveDeadNode(
N);
2181void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2184 EVT VT =
N->getValueType(0);
2192 WidenVector(*CurDAG));
2196 const EVT ResTys[] = {MVT::i64,
2199 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2202 CurDAG->getTargetConstant(LaneNo, dl,
2204 N->getOperand(NumVecs + 2),
2205 N->getOperand(NumVecs + 3),
2207 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2219 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2220 AArch64::qsub2, AArch64::qsub3 };
2221 for (
unsigned i = 0; i < NumVecs; ++i) {
2222 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2232 CurDAG->RemoveDeadNode(
N);
2235void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2238 EVT VT =
N->getOperand(2)->getValueType(0);
2246 WidenVector(*CurDAG));
2250 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2252 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2253 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2254 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2258 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2263void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2266 EVT VT =
N->getOperand(2)->getValueType(0);
2274 WidenVector(*CurDAG));
2278 const EVT ResTys[] = {MVT::i64,
2281 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2283 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2284 N->getOperand(NumVecs + 2),
2285 N->getOperand(NumVecs + 3),
2287 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2291 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2298 unsigned &LSB,
unsigned &MSB,
2299 unsigned NumberOfIgnoredLowBits,
2300 bool BiggerPattern) {
2302 "N must be a AND operation to call this function");
2304 EVT VT =
N->getValueType(0);
2309 assert((VT == MVT::i32 || VT == MVT::i64) &&
2310 "Type checking must have been done before calling this function");
2324 const SDNode *Op0 =
N->getOperand(0).getNode();
2328 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2331 if (AndImm & (AndImm + 1))
2334 bool ClampMSB =
false;
2354 ClampMSB = (VT == MVT::i32);
2355 }
else if (BiggerPattern) {
2361 Opd0 =
N->getOperand(0);
2367 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2370 <<
": Found large shift immediate, this should not happen\n"));
2376 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2377 : llvm::countr_one<uint64_t>(AndImm)) -
2384 MSB = MSB > 31 ? 31 : MSB;
2386 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2391 SDValue &Opd0,
unsigned &Immr,
2395 EVT VT =
N->getValueType(0);
2397 assert((VT == MVT::i32 || VT == MVT::i64) &&
2398 "Type checking must have been done before calling this function");
2402 Op =
Op->getOperand(0);
2403 VT =
Op->getValueType(0);
2412 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2416 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2417 Opd0 =
Op.getOperand(0);
2419 Imms = ShiftImm + Width - 1;
2447 Opd0 =
N->getOperand(0).getOperand(0);
2457 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2464 unsigned &Immr,
unsigned &Imms,
2465 bool BiggerPattern) {
2467 "N must be a SHR/SRA operation to call this function");
2469 EVT VT =
N->getValueType(0);
2474 assert((VT == MVT::i32 || VT == MVT::i64) &&
2475 "Type checking must have been done before calling this function");
2485 Opd0 =
N->getOperand(0).getOperand(0);
2486 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2492 Opd0 =
N->getOperand(0).getOperand(0);
2495 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2496 }
else if (BiggerPattern) {
2500 Opd0 =
N->getOperand(0);
2509 <<
": Found large shift immediate, this should not happen\n"));
2518 "bad amount in shift node!");
2519 int immr = SrlImm - ShlImm;
2524 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2526 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2530bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2533 EVT VT =
N->getValueType(0);
2534 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2535 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2546 unsigned Immr = ShiftImm;
2548 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2549 CurDAG->getTargetConstant(Imms, dl, VT)};
2550 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2555 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2556 unsigned NumberOfIgnoredLowBits = 0,
2557 bool BiggerPattern =
false) {
2558 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2561 switch (
N->getOpcode()) {
2563 if (!
N->isMachineOpcode())
2568 NumberOfIgnoredLowBits, BiggerPattern);
2577 unsigned NOpc =
N->getMachineOpcode();
2581 case AArch64::SBFMWri:
2582 case AArch64::UBFMWri:
2583 case AArch64::SBFMXri:
2584 case AArch64::UBFMXri:
2586 Opd0 =
N->getOperand(0);
2587 Immr =
N->getConstantOperandVal(1);
2588 Imms =
N->getConstantOperandVal(2);
2595bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2596 unsigned Opc, Immr, Imms;
2601 EVT VT =
N->getValueType(0);
2606 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2607 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2608 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2610 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2611 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2617 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2618 CurDAG->getTargetConstant(Imms, dl, VT)};
2619 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2628 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2629 assert((VT == MVT::i32 || VT == MVT::i64) &&
2630 "i32 or i64 mask type expected!");
2636 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2637 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2660 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2670 APInt OpUsefulBits(UsefulBits);
2674 OpUsefulBits <<= MSB - Imm + 1;
2679 OpUsefulBits <<= Imm;
2681 OpUsefulBits <<= MSB + 1;
2684 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2690 UsefulBits &= OpUsefulBits;
2696 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2698 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2706 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2707 APInt Mask(UsefulBits);
2708 Mask.clearAllBits();
2716 Mask.lshrInPlace(ShiftAmt);
2722 Mask.lshrInPlace(ShiftAmt);
2734 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2736 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2738 APInt OpUsefulBits(UsefulBits);
2752 OpUsefulBits <<= Width;
2755 if (
Op.getOperand(1) == Orig) {
2757 Mask = ResultUsefulBits & OpUsefulBits;
2761 if (
Op.getOperand(0) == Orig)
2763 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2769 OpUsefulBits <<= Width;
2771 OpUsefulBits <<= LSB;
2773 if (
Op.getOperand(1) == Orig) {
2775 Mask = ResultUsefulBits & OpUsefulBits;
2776 Mask.lshrInPlace(LSB);
2779 if (
Op.getOperand(0) == Orig)
2780 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2797 case AArch64::ANDSWri:
2798 case AArch64::ANDSXri:
2799 case AArch64::ANDWri:
2800 case AArch64::ANDXri:
2804 case AArch64::UBFMWri:
2805 case AArch64::UBFMXri:
2808 case AArch64::ORRWrs:
2809 case AArch64::ORRXrs:
2814 case AArch64::BFMWri:
2815 case AArch64::BFMXri:
2818 case AArch64::STRBBui:
2819 case AArch64::STURBBi:
2825 case AArch64::STRHHui:
2826 case AArch64::STURHHi:
2839 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
2841 UsefulBits =
APInt(Bitwidth, 0);
2850 UsersUsefulBits |= UsefulBitsForUse;
2855 UsefulBits &= UsersUsefulBits;
2865 EVT VT =
Op.getValueType();
2868 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2871 if (ShlAmount > 0) {
2874 UBFMOpc, dl, VT,
Op,
2879 assert(ShlAmount < 0 &&
"expected right shift");
2880 int ShrAmount = -ShlAmount;
2906 bool BiggerPattern,
SDValue &Src,
2907 int &DstLSB,
int &Width) {
2908 EVT VT =
Op.getValueType();
2917 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2921 switch (
Op.getOpcode()) {
2926 NonZeroBits, Src, DstLSB, Width);
2929 NonZeroBits, Src, DstLSB, Width);
2942 EVT VT =
Op.getValueType();
2943 assert((VT == MVT::i32 || VT == MVT::i64) &&
2944 "Caller guarantees VT is one of i32 or i64");
2957 assert((~AndImm & NonZeroBits) == 0 &&
2958 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
2987 if (!BiggerPattern && !AndOp0.
hasOneUse())
3006 <<
"Found large Width in bit-field-positioning -- this indicates no "
3007 "proper combining / constant folding was performed\n");
3016 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3031 "Op.getNode() should be a SHL node to call this function");
3033 "Op.getNode() should shift ShlImm to call this function");
3040 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3064 EVT VT =
Op.getValueType();
3065 assert((VT == MVT::i32 || VT == MVT::i64) &&
3066 "Caller guarantees that type is i32 or i64");
3073 if (!BiggerPattern && !
Op.hasOneUse())
3082 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3090 assert(VT == MVT::i32 || VT == MVT::i64);
3101 EVT VT =
N->getValueType(0);
3102 if (VT != MVT::i32 && VT != MVT::i64)
3120 if (!
And.hasOneUse() ||
3130 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3137 if ((OrImm & NotKnownZero) != 0) {
3149 unsigned ImmS = Width - 1;
3155 bool IsBFI = LSB != 0;
3160 unsigned OrChunks = 0, BFIChunks = 0;
3161 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3162 if (((OrImm >> Shift) & 0xFFFF) != 0)
3164 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3167 if (BFIChunks > OrChunks)
3173 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3181 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3190 if (!Dst.hasOneUse())
3193 EVT VT = Dst.getValueType();
3194 assert((VT == MVT::i32 || VT == MVT::i64) &&
3195 "Caller should guarantee that VT is one of i32 or i64");
3203 SDValue DstOp0 = Dst.getOperand(0);
3223 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3224 unsigned MaskWidth =
3227 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3233 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3234 ShiftedOperand =
SDValue(UBFMNode, 0);
3244 ShiftedOperand = Dst.getOperand(0);
3251 ShiftedOperand = Dst.getOperand(0);
3263 const bool BiggerPattern) {
3264 EVT VT =
N->getValueType(0);
3265 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3266 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3267 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3268 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3269 assert((VT == MVT::i32 || VT == MVT::i64) &&
3270 "Expect result type to be i32 or i64 since N is combinable to BFM");
3277 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3280 if (BiggerPattern) {
3294 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3303 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3365 EVT VT =
N->getValueType(0);
3366 if (VT != MVT::i32 && VT != MVT::i64)
3374 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3375 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3395 for (
int I = 0;
I < 4; ++
I) {
3398 unsigned ImmR, ImmS;
3399 bool BiggerPattern =
I / 2;
3400 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3402 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3408 NumberOfIgnoredLowBits, BiggerPattern)) {
3411 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3412 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3417 Width = ImmS - ImmR + 1;
3428 Src, DstLSB, Width)) {
3436 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3446 APInt BitsToBeInserted =
3449 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3473 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3506 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3508 if (Src->hasOneUse() &&
3511 Src = Src->getOperand(0);
3521 unsigned ImmS = Width - 1;
3527 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3535bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3544 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3557bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3561 EVT VT =
N->getValueType(0);
3562 if (VT != MVT::i32 && VT != MVT::i64)
3568 Op0, DstLSB, Width))
3574 unsigned ImmS = Width - 1;
3577 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3578 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3579 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3580 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3586bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3587 EVT VT =
N->getValueType(0);
3590 switch (
N->getOpcode()) {
3592 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3595 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3598 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3601 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3609 if (VT == MVT::i32) {
3612 }
else if (VT == MVT::i64) {
3618 SDValue ShiftAmt =
N->getOperand(1);
3638 (Add0Imm %
Size == 0)) {
3644 if (SubVT == MVT::i32) {
3645 NegOpc = AArch64::SUBWrr;
3646 ZeroReg = AArch64::WZR;
3648 assert(SubVT == MVT::i64);
3649 NegOpc = AArch64::SUBXrr;
3650 ZeroReg = AArch64::XZR;
3653 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3655 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3656 NewShiftAmt =
SDValue(Neg, 0);
3664 if (SubVT == MVT::i32) {
3665 NotOpc = AArch64::ORNWrr;
3666 ZeroReg = AArch64::WZR;
3668 assert(SubVT == MVT::i64);
3669 NotOpc = AArch64::ORNXrr;
3670 ZeroReg = AArch64::XZR;
3673 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3675 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3676 NewShiftAmt =
SDValue(Not, 0);
3697 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3698 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3700 AArch64::SUBREG_TO_REG,
DL, VT,
3701 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3702 NewShiftAmt =
SDValue(Ext, 0);
3705 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3706 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3713 bool isReciprocal) {
3716 FVal = CN->getValueAPF();
3717 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3720 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3724 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3725 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3748 if (!IsExact || !IntVal.isPowerOf2())
3750 unsigned FBits = IntVal.logBase2();
3754 if (FBits == 0 || FBits > RegWidth)
return false;
3760bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3761 unsigned RegWidth) {
3766bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3768 unsigned RegWidth) {
3778 RegString.
split(Fields,
':');
3780 if (Fields.
size() == 1)
3784 &&
"Invalid number of fields in read register string");
3787 bool AllIntFields =
true;
3791 AllIntFields &= !
Field.getAsInteger(10, IntField);
3796 "Unexpected non-integer value in special register string.");
3801 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3802 (Ops[3] << 3) | (Ops[4]);
3809bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3810 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3811 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3816 unsigned Opcode64Bit = AArch64::MRS;
3821 const auto *TheReg =
3823 if (TheReg && TheReg->Readable &&
3824 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3825 Imm = TheReg->Encoding;
3831 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3832 Opcode64Bit = AArch64::ADR;
3840 SDValue InChain =
N->getOperand(0);
3841 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
3842 if (!ReadIs128Bit) {
3843 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
3844 {SysRegImm, InChain});
3848 {MVT::Untyped , MVT::Other },
3849 {SysRegImm, InChain});
3853 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
3855 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
3861 ReplaceUses(
SDValue(
N, 2), OutChain);
3870bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
3871 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3872 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3877 if (!WriteIs128Bit) {
3883 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
3885 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
3886 "Expected a constant integer expression.");
3887 unsigned Reg = PMapper->Encoding;
3888 uint64_t Immed =
N->getConstantOperandVal(2);
3889 CurDAG->SelectNodeTo(
3890 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
3891 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
3897 if (trySelectPState(
3898 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
3899 AArch64::MSRpstateImm4))
3901 if (trySelectPState(
3902 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
3903 AArch64::MSRpstateImm1))
3913 if (TheReg && TheReg->Writeable &&
3914 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3915 Imm = TheReg->Encoding;
3923 SDValue InChain =
N->getOperand(0);
3924 if (!WriteIs128Bit) {
3925 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
3926 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3927 N->getOperand(2), InChain);
3931 SDNode *Pair = CurDAG->getMachineNode(
3932 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
3933 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
3936 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
3938 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
3940 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
3941 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3949bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3951 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3954 if (Subtarget->hasLSE())
return false;
3956 if (MemTy == MVT::i8)
3957 Opcode = AArch64::CMP_SWAP_8;
3958 else if (MemTy == MVT::i16)
3959 Opcode = AArch64::CMP_SWAP_16;
3960 else if (MemTy == MVT::i32)
3961 Opcode = AArch64::CMP_SWAP_32;
3962 else if (MemTy == MVT::i64)
3963 Opcode = AArch64::CMP_SWAP_64;
3967 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
3968 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3970 SDNode *CmpSwap = CurDAG->getMachineNode(
3972 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
3975 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3979 CurDAG->RemoveDeadNode(
N);
3986 if (!isa<ConstantSDNode>(
N))
3998 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
3999 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4006 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4007 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4011 if (Val <= 65280 && Val % 256 == 0) {
4012 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4013 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4024bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4027 if (!isa<ConstantSDNode>(
N))
4031 int64_t Val = cast<ConstantSDNode>(
N)
4048 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4049 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4056 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4057 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4061 if (Val <= 65280 && Val % 256 == 0) {
4062 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4063 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4076 if (!isa<ConstantSDNode>(
N))
4080 int64_t Val = cast<ConstantSDNode>(
N)
4088 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4089 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4095 if (Val >= -128 && Val <= 127) {
4096 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4097 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4101 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4102 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4103 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4114bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4115 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4116 int64_t ImmVal = CNode->getSExtValue();
4118 if (ImmVal >= -128 && ImmVal < 128) {
4119 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4127 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4128 uint64_t ImmVal = CNode->getZExtValue();
4138 ImmVal &= 0xFFFFFFFF;
4147 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4156 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4157 uint64_t ImmVal = CNode->getZExtValue();
4167 ImmVal |= ImmVal << 8;
4168 ImmVal |= ImmVal << 16;
4169 ImmVal |= ImmVal << 32;
4173 ImmVal |= ImmVal << 16;
4174 ImmVal |= ImmVal << 32;
4177 ImmVal &= 0xFFFFFFFF;
4178 ImmVal |= ImmVal << 32;
4188 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4203 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4204 uint64_t ImmVal = CN->getZExtValue();
4211 if (ImmVal >
High) {
4212 if (!AllowSaturation)
4217 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4224bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4228 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4240 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4241 SDValue FiOp = CurDAG->getTargetFrameIndex(
4243 int TagOffset =
N->getConstantOperandVal(3);
4245 SDNode *Out = CurDAG->getMachineNode(
4246 AArch64::TAGPstack,
DL, MVT::i64,
4247 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4248 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4249 ReplaceNode(
N, Out);
4253void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4254 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4255 "llvm.aarch64.tagp third argument must be an immediate");
4256 if (trySelectStackSlotTagP(
N))
4263 int TagOffset =
N->getConstantOperandVal(3);
4264 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4265 {
N->getOperand(1),
N->getOperand(2)});
4266 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4267 {
SDValue(N1, 0),
N->getOperand(2)});
4268 SDNode *N3 = CurDAG->getMachineNode(
4269 AArch64::ADDG,
DL, MVT::i64,
4270 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4271 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4275bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4279 if (
N->getConstantOperandVal(2) != 0)
4281 if (!
N->getOperand(0).isUndef())
4285 EVT VT =
N->getValueType(0);
4286 EVT InVT =
N->getOperand(1).getValueType();
4297 "Expected to insert into a packed scalable vector!");
4300 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4301 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4302 N->getOperand(1), RC));
4306bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4310 if (
N->getConstantOperandVal(1) != 0)
4314 EVT VT =
N->getValueType(0);
4315 EVT InVT =
N->getOperand(0).getValueType();
4326 "Expected to extract from a packed scalable vector!");
4329 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4330 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4331 N->getOperand(0), RC));
4335bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4340 EVT VT =
N->getValueType(0);
4360 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4361 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4368 APInt ShlAmt, ShrAmt;
4381 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4382 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4383 AArch64::XAR_ZZZI_D})) {
4384 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4390 if (!Subtarget->hasSHA3())
4412 if (ShAmt + HsAmt != 64)
4416 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4421void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4423 if (
Node->isMachineOpcode()) {
4425 Node->setNodeId(-1);
4430 EVT VT =
Node->getValueType(0);
4432 switch (
Node->getOpcode()) {
4437 if (SelectCMP_SWAP(
Node))
4443 if (tryReadRegister(
Node))
4449 if (tryWriteRegister(
Node))
4456 if (tryIndexedLoad(
Node))
4465 if (tryBitfieldExtractOp(
Node))
4467 if (tryBitfieldInsertInZeroOp(
Node))
4472 if (tryShiftAmountMod(
Node))
4477 if (tryBitfieldExtractOpFromSExt(
Node))
4482 if (tryBitfieldInsertOp(
Node))
4484 if (trySelectXAR(
Node))
4489 if (trySelectCastScalableToFixedLengthVector(
Node))
4495 if (trySelectCastFixedLengthToScalableVector(
Node))
4504 if (ConstNode->
isZero()) {
4505 if (VT == MVT::i32) {
4507 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4508 ReplaceNode(
Node,
New.getNode());
4510 }
else if (VT == MVT::i64) {
4512 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4513 ReplaceNode(
Node,
New.getNode());
4522 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4525 SDValue TFI = CurDAG->getTargetFrameIndex(
4528 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4529 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4530 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4534 unsigned IntNo =
Node->getConstantOperandVal(1);
4538 case Intrinsic::aarch64_ldaxp:
4539 case Intrinsic::aarch64_ldxp: {
4541 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4546 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4547 MVT::Other, MemAddr, Chain);
4551 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4552 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4553 ReplaceNode(
Node, Ld);
4556 case Intrinsic::aarch64_stlxp:
4557 case Intrinsic::aarch64_stxp: {
4559 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4567 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4569 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4572 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4573 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4575 ReplaceNode(
Node, St);
4578 case Intrinsic::aarch64_neon_ld1x2:
4579 if (VT == MVT::v8i8) {
4580 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4582 }
else if (VT == MVT::v16i8) {
4583 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4585 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4586 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4588 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4589 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4591 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4592 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4594 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4595 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4597 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4598 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4600 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4601 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4605 case Intrinsic::aarch64_neon_ld1x3:
4606 if (VT == MVT::v8i8) {
4607 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4609 }
else if (VT == MVT::v16i8) {
4610 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4612 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4613 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4615 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4616 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4618 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4619 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4621 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4622 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4624 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4625 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4627 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4628 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4632 case Intrinsic::aarch64_neon_ld1x4:
4633 if (VT == MVT::v8i8) {
4634 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4636 }
else if (VT == MVT::v16i8) {
4637 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4639 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4640 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4642 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4643 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4645 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4646 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4648 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4649 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4651 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4652 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4654 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4655 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4659 case Intrinsic::aarch64_neon_ld2:
4660 if (VT == MVT::v8i8) {
4661 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4663 }
else if (VT == MVT::v16i8) {
4664 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4666 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4667 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4669 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4670 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4672 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4673 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4675 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4676 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4678 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4679 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4681 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4682 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4686 case Intrinsic::aarch64_neon_ld3:
4687 if (VT == MVT::v8i8) {
4688 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4690 }
else if (VT == MVT::v16i8) {
4691 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4693 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4694 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4696 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4697 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4699 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4700 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4702 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4703 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4705 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4706 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4708 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4709 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4713 case Intrinsic::aarch64_neon_ld4:
4714 if (VT == MVT::v8i8) {
4715 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4717 }
else if (VT == MVT::v16i8) {
4718 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4720 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4721 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4723 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4724 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4726 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4727 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4729 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4730 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4732 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4733 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4735 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4736 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4740 case Intrinsic::aarch64_neon_ld2r:
4741 if (VT == MVT::v8i8) {
4742 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4744 }
else if (VT == MVT::v16i8) {
4745 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4747 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4748 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4750 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4751 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4753 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4754 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4756 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4757 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4759 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4760 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4762 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4763 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4767 case Intrinsic::aarch64_neon_ld3r:
4768 if (VT == MVT::v8i8) {
4769 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4771 }
else if (VT == MVT::v16i8) {
4772 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4774 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4775 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4777 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4778 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4780 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4781 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4783 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4784 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4786 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4787 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4789 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4790 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4794 case Intrinsic::aarch64_neon_ld4r:
4795 if (VT == MVT::v8i8) {
4796 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4798 }
else if (VT == MVT::v16i8) {
4799 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4801 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4802 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4804 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4805 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
4807 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4808 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
4810 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4811 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
4813 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4814 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
4816 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4817 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
4821 case Intrinsic::aarch64_neon_ld2lane:
4822 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4823 SelectLoadLane(
Node, 2, AArch64::LD2i8);
4825 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4826 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4827 SelectLoadLane(
Node, 2, AArch64::LD2i16);
4829 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4831 SelectLoadLane(
Node, 2, AArch64::LD2i32);
4833 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4835 SelectLoadLane(
Node, 2, AArch64::LD2i64);
4839 case Intrinsic::aarch64_neon_ld3lane:
4840 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4841 SelectLoadLane(
Node, 3, AArch64::LD3i8);
4843 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4844 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4845 SelectLoadLane(
Node, 3, AArch64::LD3i16);
4847 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4849 SelectLoadLane(
Node, 3, AArch64::LD3i32);
4851 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4853 SelectLoadLane(
Node, 3, AArch64::LD3i64);
4857 case Intrinsic::aarch64_neon_ld4lane:
4858 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4859 SelectLoadLane(
Node, 4, AArch64::LD4i8);
4861 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4862 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4863 SelectLoadLane(
Node, 4, AArch64::LD4i16);
4865 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4867 SelectLoadLane(
Node, 4, AArch64::LD4i32);
4869 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4871 SelectLoadLane(
Node, 4, AArch64::LD4i64);
4875 case Intrinsic::aarch64_ld64b:
4876 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
4878 case Intrinsic::aarch64_sve_ld2q_sret: {
4879 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
4882 case Intrinsic::aarch64_sve_ld3q_sret: {
4883 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
4886 case Intrinsic::aarch64_sve_ld4q_sret: {
4887 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
4890 case Intrinsic::aarch64_sve_ld2_sret: {
4891 if (VT == MVT::nxv16i8) {
4892 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
4895 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4896 VT == MVT::nxv8bf16) {
4897 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
4900 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4901 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
4904 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4905 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
4911 case Intrinsic::aarch64_sve_ld1_pn_x2: {
4912 if (VT == MVT::nxv16i8) {
4913 if (Subtarget->hasSME2())
4914 SelectContiguousMultiVectorLoad(
4915 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
4916 else if (Subtarget->hasSVE2p1())
4917 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
4922 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4923 VT == MVT::nxv8bf16) {
4924 if (Subtarget->hasSME2())
4925 SelectContiguousMultiVectorLoad(
4926 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
4927 else if (Subtarget->hasSVE2p1())
4928 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
4933 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4934 if (Subtarget->hasSME2())
4935 SelectContiguousMultiVectorLoad(
4936 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
4937 else if (Subtarget->hasSVE2p1())
4938 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
4943 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4944 if (Subtarget->hasSME2())
4945 SelectContiguousMultiVectorLoad(
4946 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
4947 else if (Subtarget->hasSVE2p1())
4948 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
4956 case Intrinsic::aarch64_sve_ld1_pn_x4: {
4957 if (VT == MVT::nxv16i8) {
4958 if (Subtarget->hasSME2())
4959 SelectContiguousMultiVectorLoad(
4960 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
4961 else if (Subtarget->hasSVE2p1())
4962 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
4967 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4968 VT == MVT::nxv8bf16) {
4969 if (Subtarget->hasSME2())
4970 SelectContiguousMultiVectorLoad(
4971 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
4972 else if (Subtarget->hasSVE2p1())
4973 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
4978 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4979 if (Subtarget->hasSME2())
4980 SelectContiguousMultiVectorLoad(
4981 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
4982 else if (Subtarget->hasSVE2p1())
4983 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
4988 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4989 if (Subtarget->hasSME2())
4990 SelectContiguousMultiVectorLoad(
4991 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
4992 else if (Subtarget->hasSVE2p1())
4993 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5001 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5002 if (VT == MVT::nxv16i8) {
5003 if (Subtarget->hasSME2())
5004 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5005 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5006 AArch64::LDNT1B_2Z_PSEUDO);
5007 else if (Subtarget->hasSVE2p1())
5008 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5009 AArch64::LDNT1B_2Z);
5013 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5014 VT == MVT::nxv8bf16) {
5015 if (Subtarget->hasSME2())
5016 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5017 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5018 AArch64::LDNT1H_2Z_PSEUDO);
5019 else if (Subtarget->hasSVE2p1())
5020 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5021 AArch64::LDNT1H_2Z);
5025 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5026 if (Subtarget->hasSME2())
5027 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5028 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5029 AArch64::LDNT1W_2Z_PSEUDO);
5030 else if (Subtarget->hasSVE2p1())
5031 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5032 AArch64::LDNT1W_2Z);
5036 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5037 if (Subtarget->hasSME2())
5038 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5039 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5040 AArch64::LDNT1D_2Z_PSEUDO);
5041 else if (Subtarget->hasSVE2p1())
5042 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5043 AArch64::LDNT1D_2Z);
5050 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5051 if (VT == MVT::nxv16i8) {
5052 if (Subtarget->hasSME2())
5053 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5054 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5055 AArch64::LDNT1B_4Z_PSEUDO);
5056 else if (Subtarget->hasSVE2p1())
5057 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5058 AArch64::LDNT1B_4Z);
5062 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5063 VT == MVT::nxv8bf16) {
5064 if (Subtarget->hasSME2())
5065 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5066 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5067 AArch64::LDNT1H_4Z_PSEUDO);
5068 else if (Subtarget->hasSVE2p1())
5069 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5070 AArch64::LDNT1H_4Z);
5074 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5075 if (Subtarget->hasSME2())
5076 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5077 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5078 AArch64::LDNT1W_4Z_PSEUDO);
5079 else if (Subtarget->hasSVE2p1())
5080 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5081 AArch64::LDNT1W_4Z);
5085 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5086 if (Subtarget->hasSME2())
5087 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5088 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5089 AArch64::LDNT1D_4Z_PSEUDO);
5090 else if (Subtarget->hasSVE2p1())
5091 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5092 AArch64::LDNT1D_4Z);
5099 case Intrinsic::aarch64_sve_ld3_sret: {
5100 if (VT == MVT::nxv16i8) {
5101 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5104 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5105 VT == MVT::nxv8bf16) {
5106 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5109 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5110 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5113 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5114 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5120 case Intrinsic::aarch64_sve_ld4_sret: {
5121 if (VT == MVT::nxv16i8) {
5122 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5125 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5126 VT == MVT::nxv8bf16) {
5127 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5130 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5131 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5134 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5135 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5141 case Intrinsic::aarch64_sme_read_hor_vg2: {
5142 if (VT == MVT::nxv16i8) {
5143 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5144 AArch64::MOVA_2ZMXI_H_B);
5146 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5147 VT == MVT::nxv8bf16) {
5148 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5149 AArch64::MOVA_2ZMXI_H_H);
5151 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5152 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5153 AArch64::MOVA_2ZMXI_H_S);
5155 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5156 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5157 AArch64::MOVA_2ZMXI_H_D);
5162 case Intrinsic::aarch64_sme_read_ver_vg2: {
5163 if (VT == MVT::nxv16i8) {
5164 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5165 AArch64::MOVA_2ZMXI_V_B);
5167 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5168 VT == MVT::nxv8bf16) {
5169 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5170 AArch64::MOVA_2ZMXI_V_H);
5172 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5173 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5174 AArch64::MOVA_2ZMXI_V_S);
5176 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5177 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5178 AArch64::MOVA_2ZMXI_V_D);
5183 case Intrinsic::aarch64_sme_read_hor_vg4: {
5184 if (VT == MVT::nxv16i8) {
5185 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5186 AArch64::MOVA_4ZMXI_H_B);
5188 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5189 VT == MVT::nxv8bf16) {
5190 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5191 AArch64::MOVA_4ZMXI_H_H);
5193 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5194 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5195 AArch64::MOVA_4ZMXI_H_S);
5197 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5198 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5199 AArch64::MOVA_4ZMXI_H_D);
5204 case Intrinsic::aarch64_sme_read_ver_vg4: {
5205 if (VT == MVT::nxv16i8) {
5206 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5207 AArch64::MOVA_4ZMXI_V_B);
5209 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5210 VT == MVT::nxv8bf16) {
5211 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5212 AArch64::MOVA_4ZMXI_V_H);
5214 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5215 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5216 AArch64::MOVA_4ZMXI_V_S);
5218 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5219 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5220 AArch64::MOVA_4ZMXI_V_D);
5225 case Intrinsic::aarch64_sme_read_vg1x2: {
5226 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5227 AArch64::MOVA_VG2_2ZMXI);
5230 case Intrinsic::aarch64_sme_read_vg1x4: {
5231 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5232 AArch64::MOVA_VG4_4ZMXI);
5235 case Intrinsic::swift_async_context_addr: {
5238 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5240 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5241 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5242 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5246 CurDAG->RemoveDeadNode(
Node);
5248 auto &MF = CurDAG->getMachineFunction();
5249 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5253 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5254 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5255 Node->getValueType(0),
5256 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5257 AArch64::LUTI2_4ZTZI_S}))
5259 SelectMultiVectorLuti(
Node, 4, Opc, 3);
5262 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5263 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5264 Node->getValueType(0),
5265 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5267 SelectMultiVectorLuti(
Node, 4, Opc, 1);
5270 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5271 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5272 Node->getValueType(0),
5273 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5274 AArch64::LUTI2_2ZTZI_S}))
5276 SelectMultiVectorLuti(
Node, 2, Opc, 7);
5279 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5280 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5281 Node->getValueType(0),
5282 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5283 AArch64::LUTI4_2ZTZI_S}))
5285 SelectMultiVectorLuti(
Node, 2, Opc, 3);
5291 unsigned IntNo =
Node->getConstantOperandVal(0);
5295 case Intrinsic::aarch64_tagp:
5298 case Intrinsic::aarch64_neon_tbl2:
5299 SelectTable(
Node, 2,
5300 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5303 case Intrinsic::aarch64_neon_tbl3:
5304 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5305 : AArch64::TBLv16i8Three,
5308 case Intrinsic::aarch64_neon_tbl4:
5309 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5310 : AArch64::TBLv16i8Four,
5313 case Intrinsic::aarch64_neon_tbx2:
5314 SelectTable(
Node, 2,
5315 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5318 case Intrinsic::aarch64_neon_tbx3:
5319 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5320 : AArch64::TBXv16i8Three,
5323 case Intrinsic::aarch64_neon_tbx4:
5324 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5325 : AArch64::TBXv16i8Four,
5328 case Intrinsic::aarch64_sve_srshl_single_x2:
5329 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5330 Node->getValueType(0),
5331 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5332 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5333 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5335 case Intrinsic::aarch64_sve_srshl_single_x4:
5336 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5337 Node->getValueType(0),
5338 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5339 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5340 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5342 case Intrinsic::aarch64_sve_urshl_single_x2:
5343 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5344 Node->getValueType(0),
5345 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5346 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5347 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5349 case Intrinsic::aarch64_sve_urshl_single_x4:
5350 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5351 Node->getValueType(0),
5352 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5353 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5354 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5356 case Intrinsic::aarch64_sve_srshl_x2:
5357 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5358 Node->getValueType(0),
5359 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5360 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5361 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5363 case Intrinsic::aarch64_sve_srshl_x4:
5364 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5365 Node->getValueType(0),
5366 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5367 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5368 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5370 case Intrinsic::aarch64_sve_urshl_x2:
5371 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5372 Node->getValueType(0),
5373 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5374 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5375 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5377 case Intrinsic::aarch64_sve_urshl_x4:
5378 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5379 Node->getValueType(0),
5380 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5381 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5382 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5384 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5385 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5386 Node->getValueType(0),
5387 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5388 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5389 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5391 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5392 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5393 Node->getValueType(0),
5394 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5395 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5396 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5398 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5399 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5400 Node->getValueType(0),
5401 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5402 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5403 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5405 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5406 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5407 Node->getValueType(0),
5408 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5409 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5410 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5412 case Intrinsic::aarch64_sve_whilege_x2:
5413 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5414 Node->getValueType(0),
5415 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5416 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5417 SelectWhilePair(
Node,
Op);
5419 case Intrinsic::aarch64_sve_whilegt_x2:
5420 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5421 Node->getValueType(0),
5422 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5423 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5424 SelectWhilePair(
Node,
Op);
5426 case Intrinsic::aarch64_sve_whilehi_x2:
5427 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5428 Node->getValueType(0),
5429 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5430 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5431 SelectWhilePair(
Node,
Op);
5433 case Intrinsic::aarch64_sve_whilehs_x2:
5434 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5435 Node->getValueType(0),
5436 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5437 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5438 SelectWhilePair(
Node,
Op);
5440 case Intrinsic::aarch64_sve_whilele_x2:
5441 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5442 Node->getValueType(0),
5443 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5444 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5445 SelectWhilePair(
Node,
Op);
5447 case Intrinsic::aarch64_sve_whilelo_x2:
5448 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5449 Node->getValueType(0),
5450 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5451 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5452 SelectWhilePair(
Node,
Op);
5454 case Intrinsic::aarch64_sve_whilels_x2:
5455 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5456 Node->getValueType(0),
5457 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5458 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5459 SelectWhilePair(
Node,
Op);
5461 case Intrinsic::aarch64_sve_whilelt_x2:
5462 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5463 Node->getValueType(0),
5464 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5465 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5466 SelectWhilePair(
Node,
Op);
5468 case Intrinsic::aarch64_sve_smax_single_x2:
5469 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5470 Node->getValueType(0),
5471 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5472 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5473 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5475 case Intrinsic::aarch64_sve_umax_single_x2:
5476 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5477 Node->getValueType(0),
5478 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5479 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5480 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5482 case Intrinsic::aarch64_sve_fmax_single_x2:
5483 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5484 Node->getValueType(0),
5485 {0, AArch64::FMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_S,
5486 AArch64::FMAX_VG2_2ZZ_D}))
5487 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5489 case Intrinsic::aarch64_sve_smax_single_x4:
5490 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5491 Node->getValueType(0),
5492 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5493 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5494 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5496 case Intrinsic::aarch64_sve_umax_single_x4:
5497 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5498 Node->getValueType(0),
5499 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5500 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5501 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5503 case Intrinsic::aarch64_sve_fmax_single_x4:
5504 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5505 Node->getValueType(0),
5506 {0, AArch64::FMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_S,
5507 AArch64::FMAX_VG4_4ZZ_D}))
5508 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5510 case Intrinsic::aarch64_sve_smin_single_x2:
5511 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5512 Node->getValueType(0),
5513 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5514 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5515 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5517 case Intrinsic::aarch64_sve_umin_single_x2:
5518 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5519 Node->getValueType(0),
5520 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5521 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5522 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5524 case Intrinsic::aarch64_sve_fmin_single_x2:
5525 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5526 Node->getValueType(0),
5527 {0, AArch64::FMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_S,
5528 AArch64::FMIN_VG2_2ZZ_D}))
5529 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5531 case Intrinsic::aarch64_sve_smin_single_x4:
5532 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5533 Node->getValueType(0),
5534 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5535 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5536 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5538 case Intrinsic::aarch64_sve_umin_single_x4:
5539 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5540 Node->getValueType(0),
5541 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5542 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5543 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5545 case Intrinsic::aarch64_sve_fmin_single_x4:
5546 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5547 Node->getValueType(0),
5548 {0, AArch64::FMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_S,
5549 AArch64::FMIN_VG4_4ZZ_D}))
5550 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5552 case Intrinsic::aarch64_sve_smax_x2:
5553 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5554 Node->getValueType(0),
5555 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5556 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5557 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5559 case Intrinsic::aarch64_sve_umax_x2:
5560 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5561 Node->getValueType(0),
5562 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5563 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5564 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5566 case Intrinsic::aarch64_sve_fmax_x2:
5567 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5568 Node->getValueType(0),
5569 {0, AArch64::FMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_S,
5570 AArch64::FMAX_VG2_2Z2Z_D}))
5571 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5573 case Intrinsic::aarch64_sve_smax_x4:
5574 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5575 Node->getValueType(0),
5576 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5577 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5578 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5580 case Intrinsic::aarch64_sve_umax_x4:
5581 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5582 Node->getValueType(0),
5583 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5584 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5585 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5587 case Intrinsic::aarch64_sve_fmax_x4:
5588 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5589 Node->getValueType(0),
5590 {0, AArch64::FMAX_VG4_4Z4Z_H, AArch64::FMAX_VG4_4Z4Z_S,
5591 AArch64::FMAX_VG4_4Z4Z_D}))
5592 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5594 case Intrinsic::aarch64_sve_smin_x2:
5595 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5596 Node->getValueType(0),
5597 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5598 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5599 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5601 case Intrinsic::aarch64_sve_umin_x2:
5602 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5603 Node->getValueType(0),
5604 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5605 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5606 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5608 case Intrinsic::aarch64_sve_fmin_x2:
5609 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5610 Node->getValueType(0),
5611 {0, AArch64::FMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_S,
5612 AArch64::FMIN_VG2_2Z2Z_D}))
5613 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5615 case Intrinsic::aarch64_sve_smin_x4:
5616 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5617 Node->getValueType(0),
5618 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5619 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5620 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5622 case Intrinsic::aarch64_sve_umin_x4:
5623 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5624 Node->getValueType(0),
5625 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
5626 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
5627 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5629 case Intrinsic::aarch64_sve_fmin_x4:
5630 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5631 Node->getValueType(0),
5632 {0, AArch64::FMIN_VG4_4Z4Z_H, AArch64::FMIN_VG4_4Z4Z_S,
5633 AArch64::FMIN_VG4_4Z4Z_D}))
5634 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5636 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
5637 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5638 Node->getValueType(0),
5639 {0, AArch64::FMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_S,
5640 AArch64::FMAXNM_VG2_2ZZ_D}))
5641 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5643 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
5644 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5645 Node->getValueType(0),
5646 {0, AArch64::FMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_S,
5647 AArch64::FMAXNM_VG4_4ZZ_D}))
5648 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5650 case Intrinsic::aarch64_sve_fminnm_single_x2:
5651 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5652 Node->getValueType(0),
5653 {0, AArch64::FMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_S,
5654 AArch64::FMINNM_VG2_2ZZ_D}))
5655 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5657 case Intrinsic::aarch64_sve_fminnm_single_x4:
5658 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5659 Node->getValueType(0),
5660 {0, AArch64::FMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_S,
5661 AArch64::FMINNM_VG4_4ZZ_D}))
5662 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5664 case Intrinsic::aarch64_sve_fmaxnm_x2:
5665 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5666 Node->getValueType(0),
5667 {0, AArch64::FMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_S,
5668 AArch64::FMAXNM_VG2_2Z2Z_D}))
5669 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5671 case Intrinsic::aarch64_sve_fmaxnm_x4:
5672 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5673 Node->getValueType(0),
5674 {0, AArch64::FMAXNM_VG4_4Z4Z_H, AArch64::FMAXNM_VG4_4Z4Z_S,
5675 AArch64::FMAXNM_VG4_4Z4Z_D}))
5676 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5678 case Intrinsic::aarch64_sve_fminnm_x2:
5679 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5680 Node->getValueType(0),
5681 {0, AArch64::FMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_S,
5682 AArch64::FMINNM_VG2_2Z2Z_D}))
5683 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5685 case Intrinsic::aarch64_sve_fminnm_x4:
5686 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5687 Node->getValueType(0),
5688 {0, AArch64::FMINNM_VG4_4Z4Z_H, AArch64::FMINNM_VG4_4Z4Z_S,
5689 AArch64::FMINNM_VG4_4Z4Z_D}))
5690 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5692 case Intrinsic::aarch64_sve_fcvtzs_x2:
5693 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
5695 case Intrinsic::aarch64_sve_scvtf_x2:
5696 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
5698 case Intrinsic::aarch64_sve_fcvtzu_x2:
5699 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
5701 case Intrinsic::aarch64_sve_ucvtf_x2:
5702 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
5704 case Intrinsic::aarch64_sve_fcvtzs_x4:
5705 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
5707 case Intrinsic::aarch64_sve_scvtf_x4:
5708 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
5710 case Intrinsic::aarch64_sve_fcvtzu_x4:
5711 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
5713 case Intrinsic::aarch64_sve_ucvtf_x4:
5714 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
5716 case Intrinsic::aarch64_sve_sclamp_single_x2:
5717 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5718 Node->getValueType(0),
5719 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
5720 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
5721 SelectClamp(
Node, 2,
Op);
5723 case Intrinsic::aarch64_sve_uclamp_single_x2:
5724 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5725 Node->getValueType(0),
5726 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
5727 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
5728 SelectClamp(
Node, 2,
Op);
5730 case Intrinsic::aarch64_sve_fclamp_single_x2:
5731 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5732 Node->getValueType(0),
5733 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
5734 AArch64::FCLAMP_VG2_2Z2Z_D}))
5735 SelectClamp(
Node, 2,
Op);
5737 case Intrinsic::aarch64_sve_sclamp_single_x4:
5738 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5739 Node->getValueType(0),
5740 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
5741 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
5742 SelectClamp(
Node, 4,
Op);
5744 case Intrinsic::aarch64_sve_uclamp_single_x4:
5745 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5746 Node->getValueType(0),
5747 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
5748 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
5749 SelectClamp(
Node, 4,
Op);
5751 case Intrinsic::aarch64_sve_fclamp_single_x4:
5752 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5753 Node->getValueType(0),
5754 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
5755 AArch64::FCLAMP_VG4_4Z4Z_D}))
5756 SelectClamp(
Node, 4,
Op);
5758 case Intrinsic::aarch64_sve_add_single_x2:
5759 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5760 Node->getValueType(0),
5761 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
5762 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
5763 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5765 case Intrinsic::aarch64_sve_add_single_x4:
5766 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5767 Node->getValueType(0),
5768 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
5769 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
5770 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5772 case Intrinsic::aarch64_sve_zip_x2:
5773 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5774 Node->getValueType(0),
5775 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
5776 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
5777 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5779 case Intrinsic::aarch64_sve_zipq_x2:
5780 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5781 AArch64::ZIP_VG2_2ZZZ_Q);
5783 case Intrinsic::aarch64_sve_zip_x4:
5784 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5785 Node->getValueType(0),
5786 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
5787 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
5788 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5790 case Intrinsic::aarch64_sve_zipq_x4:
5791 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5792 AArch64::ZIP_VG4_4Z4Z_Q);
5794 case Intrinsic::aarch64_sve_uzp_x2:
5795 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5796 Node->getValueType(0),
5797 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
5798 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
5799 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5801 case Intrinsic::aarch64_sve_uzpq_x2:
5802 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5803 AArch64::UZP_VG2_2ZZZ_Q);
5805 case Intrinsic::aarch64_sve_uzp_x4:
5806 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5807 Node->getValueType(0),
5808 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
5809 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
5810 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5812 case Intrinsic::aarch64_sve_uzpq_x4:
5813 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5814 AArch64::UZP_VG4_4Z4Z_Q);
5816 case Intrinsic::aarch64_sve_sel_x2:
5817 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5818 Node->getValueType(0),
5819 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
5820 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
5821 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
5823 case Intrinsic::aarch64_sve_sel_x4:
5824 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5825 Node->getValueType(0),
5826 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
5827 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
5828 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
5830 case Intrinsic::aarch64_sve_frinta_x2:
5831 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
5833 case Intrinsic::aarch64_sve_frinta_x4:
5834 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
5836 case Intrinsic::aarch64_sve_frintm_x2:
5837 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
5839 case Intrinsic::aarch64_sve_frintm_x4:
5840 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
5842 case Intrinsic::aarch64_sve_frintn_x2:
5843 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
5845 case Intrinsic::aarch64_sve_frintn_x4:
5846 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
5848 case Intrinsic::aarch64_sve_frintp_x2:
5849 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
5851 case Intrinsic::aarch64_sve_frintp_x4:
5852 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
5854 case Intrinsic::aarch64_sve_sunpk_x2:
5855 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5856 Node->getValueType(0),
5857 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
5858 AArch64::SUNPK_VG2_2ZZ_D}))
5859 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5861 case Intrinsic::aarch64_sve_uunpk_x2:
5862 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5863 Node->getValueType(0),
5864 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
5865 AArch64::UUNPK_VG2_2ZZ_D}))
5866 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5868 case Intrinsic::aarch64_sve_sunpk_x4:
5869 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5870 Node->getValueType(0),
5871 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
5872 AArch64::SUNPK_VG4_4Z2Z_D}))
5873 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5875 case Intrinsic::aarch64_sve_uunpk_x4:
5876 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5877 Node->getValueType(0),
5878 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
5879 AArch64::UUNPK_VG4_4Z2Z_D}))
5880 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5882 case Intrinsic::aarch64_sve_pext_x2: {
5883 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5884 Node->getValueType(0),
5885 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
5886 AArch64::PEXT_2PCI_D}))
5887 SelectPExtPair(
Node,
Op);
5894 unsigned IntNo =
Node->getConstantOperandVal(1);
5895 if (
Node->getNumOperands() >= 3)
5896 VT =
Node->getOperand(2)->getValueType(0);
5900 case Intrinsic::aarch64_neon_st1x2: {
5901 if (VT == MVT::v8i8) {
5902 SelectStore(
Node, 2, AArch64::ST1Twov8b);
5904 }
else if (VT == MVT::v16i8) {
5905 SelectStore(
Node, 2, AArch64::ST1Twov16b);
5907 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5908 VT == MVT::v4bf16) {
5909 SelectStore(
Node, 2, AArch64::ST1Twov4h);
5911 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
5912 VT == MVT::v8bf16) {
5913 SelectStore(
Node, 2, AArch64::ST1Twov8h);
5915 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5916 SelectStore(
Node, 2, AArch64::ST1Twov2s);
5918 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5919 SelectStore(
Node, 2, AArch64::ST1Twov4s);
5921 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5922 SelectStore(
Node, 2, AArch64::ST1Twov2d);
5924 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5925 SelectStore(
Node, 2, AArch64::ST1Twov1d);
5930 case Intrinsic::aarch64_neon_st1x3: {
5931 if (VT == MVT::v8i8) {
5932 SelectStore(
Node, 3, AArch64::ST1Threev8b);
5934 }
else if (VT == MVT::v16i8) {
5935 SelectStore(
Node, 3, AArch64::ST1Threev16b);
5937 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5938 VT == MVT::v4bf16) {
5939 SelectStore(
Node, 3, AArch64::ST1Threev4h);
5941 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
5942 VT == MVT::v8bf16) {
5943 SelectStore(
Node, 3, AArch64::ST1Threev8h);
5945 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5946 SelectStore(
Node, 3, AArch64::ST1Threev2s);
5948 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5949 SelectStore(
Node, 3, AArch64::ST1Threev4s);
5951 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5952 SelectStore(
Node, 3, AArch64::ST1Threev2d);
5954 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5955 SelectStore(
Node, 3, AArch64::ST1Threev1d);
5960 case Intrinsic::aarch64_neon_st1x4: {
5961 if (VT == MVT::v8i8) {
5962 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
5964 }
else if (VT == MVT::v16i8) {
5965 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
5967 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5968 VT == MVT::v4bf16) {
5969 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
5971 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
5972 VT == MVT::v8bf16) {
5973 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
5975 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5976 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
5978 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5979 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
5981 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5982 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
5984 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5985 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
5990 case Intrinsic::aarch64_neon_st2: {
5991 if (VT == MVT::v8i8) {
5992 SelectStore(
Node, 2, AArch64::ST2Twov8b);
5994 }
else if (VT == MVT::v16i8) {
5995 SelectStore(
Node, 2, AArch64::ST2Twov16b);
5997 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5998 VT == MVT::v4bf16) {
5999 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6001 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6002 VT == MVT::v8bf16) {
6003 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6005 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6006 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6008 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6009 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6011 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6012 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6014 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6015 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6020 case Intrinsic::aarch64_neon_st3: {
6021 if (VT == MVT::v8i8) {
6022 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6024 }
else if (VT == MVT::v16i8) {
6025 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6027 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6028 VT == MVT::v4bf16) {
6029 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6031 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6032 VT == MVT::v8bf16) {
6033 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6035 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6036 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6038 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6039 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6041 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6042 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6044 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6045 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6050 case Intrinsic::aarch64_neon_st4: {
6051 if (VT == MVT::v8i8) {
6052 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6054 }
else if (VT == MVT::v16i8) {
6055 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6057 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6058 VT == MVT::v4bf16) {
6059 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6061 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6062 VT == MVT::v8bf16) {
6063 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6065 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6066 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6068 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6069 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6071 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6072 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6074 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6075 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6080 case Intrinsic::aarch64_neon_st2lane: {
6081 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6082 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6084 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6085 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6086 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6088 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6090 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6092 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6094 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6099 case Intrinsic::aarch64_neon_st3lane: {
6100 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6101 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6103 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6104 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6105 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6107 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6109 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6111 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6113 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6118 case Intrinsic::aarch64_neon_st4lane: {
6119 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6120 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6122 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6123 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6124 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6126 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6128 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6130 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6132 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6137 case Intrinsic::aarch64_sve_st2q: {
6138 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6141 case Intrinsic::aarch64_sve_st3q: {
6142 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6145 case Intrinsic::aarch64_sve_st4q: {
6146 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6149 case Intrinsic::aarch64_sve_st2: {
6150 if (VT == MVT::nxv16i8) {
6151 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6153 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6154 VT == MVT::nxv8bf16) {
6155 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6157 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6158 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6160 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6161 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6166 case Intrinsic::aarch64_sve_st3: {
6167 if (VT == MVT::nxv16i8) {
6168 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6170 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6171 VT == MVT::nxv8bf16) {
6172 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6174 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6175 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6177 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6178 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6183 case Intrinsic::aarch64_sve_st4: {
6184 if (VT == MVT::nxv16i8) {
6185 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6187 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6188 VT == MVT::nxv8bf16) {
6189 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6191 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6192 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6194 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6195 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6204 if (VT == MVT::v8i8) {
6205 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6207 }
else if (VT == MVT::v16i8) {
6208 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6210 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6211 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6213 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6214 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6216 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6217 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6219 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6220 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6222 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6223 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6225 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6226 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6232 if (VT == MVT::v8i8) {
6233 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6235 }
else if (VT == MVT::v16i8) {
6236 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6238 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6239 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6241 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6242 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6244 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6245 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6247 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6248 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6250 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6251 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6253 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6254 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6260 if (VT == MVT::v8i8) {
6261 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6263 }
else if (VT == MVT::v16i8) {
6264 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6266 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6267 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6269 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6270 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6272 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6273 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6275 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6276 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6278 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6279 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6281 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6282 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6288 if (VT == MVT::v8i8) {
6289 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6291 }
else if (VT == MVT::v16i8) {
6292 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6294 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6295 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6297 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6298 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6300 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6301 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6303 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6304 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6306 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6307 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6309 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6310 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6316 if (VT == MVT::v8i8) {
6317 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6319 }
else if (VT == MVT::v16i8) {
6320 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6322 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6323 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6325 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6326 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6328 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6329 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6331 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6332 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6334 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6335 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6337 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6338 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6344 if (VT == MVT::v8i8) {
6345 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6347 }
else if (VT == MVT::v16i8) {
6348 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6350 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6351 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6353 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6354 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6356 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6357 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6359 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6360 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6362 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6363 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6365 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6366 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6372 if (VT == MVT::v8i8) {
6373 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6375 }
else if (VT == MVT::v16i8) {
6376 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6378 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6379 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6381 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6382 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6384 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6385 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6387 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6388 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6390 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6391 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6393 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6394 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6400 if (VT == MVT::v8i8) {
6401 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6403 }
else if (VT == MVT::v16i8) {
6404 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6406 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6407 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6409 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6410 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6412 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6413 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6415 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6416 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6418 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6419 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6421 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6422 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6428 if (VT == MVT::v8i8) {
6429 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6431 }
else if (VT == MVT::v16i8) {
6432 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6434 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6435 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6437 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6438 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6440 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6441 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6443 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6444 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6446 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6447 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6449 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6450 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6456 if (VT == MVT::v8i8) {
6457 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6459 }
else if (VT == MVT::v16i8) {
6460 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6462 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6463 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6465 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6466 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6468 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6469 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6471 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6472 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6474 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6475 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6477 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6478 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6484 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6485 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6487 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6488 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6489 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6491 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6493 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6495 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6497 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6503 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6504 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6506 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6507 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6508 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6510 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6512 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6514 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6516 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6522 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6523 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6525 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6526 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6527 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6529 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6531 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6533 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6535 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6541 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6542 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6544 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6545 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6546 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6548 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6550 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6552 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6554 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6560 VT =
Node->getOperand(1).getValueType();
6561 if (VT == MVT::v8i8) {
6562 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6564 }
else if (VT == MVT::v16i8) {
6565 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6567 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6568 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6570 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6571 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6573 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6574 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6576 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6577 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6579 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6580 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6582 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6583 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6589 VT =
Node->getOperand(1).getValueType();
6590 if (VT == MVT::v8i8) {
6591 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6593 }
else if (VT == MVT::v16i8) {
6594 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6596 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6597 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6599 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6600 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6602 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6603 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6605 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6606 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6608 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6609 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
6611 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6612 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6618 VT =
Node->getOperand(1).getValueType();
6619 if (VT == MVT::v8i8) {
6620 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
6622 }
else if (VT == MVT::v16i8) {
6623 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
6625 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6626 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
6628 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6629 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
6631 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6632 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
6634 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6635 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
6637 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6638 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
6640 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6641 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6647 VT =
Node->getOperand(1).getValueType();
6648 if (VT == MVT::v8i8) {
6649 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
6651 }
else if (VT == MVT::v16i8) {
6652 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
6654 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6655 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
6657 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6658 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
6660 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6661 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
6663 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6664 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
6666 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6667 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6669 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6670 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
6676 VT =
Node->getOperand(1).getValueType();
6677 if (VT == MVT::v8i8) {
6678 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
6680 }
else if (VT == MVT::v16i8) {
6681 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
6683 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6684 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
6686 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
6687 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
6689 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6690 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
6692 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6693 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
6695 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6696 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6698 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6699 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
6705 VT =
Node->getOperand(1).getValueType();
6706 if (VT == MVT::v8i8) {
6707 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
6709 }
else if (VT == MVT::v16i8) {
6710 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
6712 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6713 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
6715 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6716 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
6718 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6719 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
6721 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6722 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
6724 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6725 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6727 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6728 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
6734 VT =
Node->getOperand(1).getValueType();
6735 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6736 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
6738 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6739 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6740 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
6742 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6744 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
6746 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6748 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
6754 VT =
Node->getOperand(1).getValueType();
6755 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6756 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
6758 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6759 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6760 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
6762 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6764 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
6766 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6768 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
6774 VT =
Node->getOperand(1).getValueType();
6775 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6776 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
6778 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6779 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6780 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
6782 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6784 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
6786 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6788 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
6794 if (VT == MVT::nxv16i8) {
6795 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
6797 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6798 VT == MVT::nxv8bf16) {
6799 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
6801 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6802 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
6804 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6805 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
6811 if (VT == MVT::nxv16i8) {
6812 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
6814 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6815 VT == MVT::nxv8bf16) {
6816 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
6818 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6819 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
6821 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6822 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
6828 if (VT == MVT::nxv16i8) {
6829 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
6831 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6832 VT == MVT::nxv8bf16) {
6833 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
6835 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6836 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
6838 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6839 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
6854 return new AArch64DAGToDAGISel(
TM, OptLevel);
6866 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
6870 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
6871 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
6885 if (isa<MemSDNode>(Root))
6886 return cast<MemSDNode>(Root)->getMemoryVT();
6888 if (isa<MemIntrinsicSDNode>(Root))
6889 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
6891 const unsigned Opcode = Root->
getOpcode();
6899 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
6901 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
6921 case Intrinsic::aarch64_sme_ldr:
6922 case Intrinsic::aarch64_sme_str:
6923 return MVT::nxv16i8;
6924 case Intrinsic::aarch64_sve_prf:
6929 case Intrinsic::aarch64_sve_ld2_sret:
6930 case Intrinsic::aarch64_sve_ld2q_sret:
6933 case Intrinsic::aarch64_sve_st2q:
6936 case Intrinsic::aarch64_sve_ld3_sret:
6937 case Intrinsic::aarch64_sve_ld3q_sret:
6940 case Intrinsic::aarch64_sve_st3q:
6943 case Intrinsic::aarch64_sve_ld4_sret:
6944 case Intrinsic::aarch64_sve_ld4q_sret:
6947 case Intrinsic::aarch64_sve_st4q:
6950 case Intrinsic::aarch64_sve_ld1udq:
6951 case Intrinsic::aarch64_sve_st1dq:
6952 return EVT(MVT::nxv1i64);
6953 case Intrinsic::aarch64_sve_ld1uwq:
6954 case Intrinsic::aarch64_sve_st1wq:
6955 return EVT(MVT::nxv1i32);
6962template <
int64_t Min,
int64_t Max>
6963bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
6971 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
6976 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
6995 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
6997 if ((MulImm % MemWidthBytes) != 0)
7000 int64_t
Offset = MulImm / MemWidthBytes;
7001 if (Offset < Min || Offset > Max)
7004 Base =
N.getOperand(0);
7006 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7013 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7019bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7037 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7038 int64_t ImmOff =
C->getSExtValue();
7039 unsigned Size = 1 << Scale;
7048 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7050 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7060 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7061 if (
C->getZExtValue() == Scale) {
7070bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7077bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7078 EVT VT =
N.getValueType();
7082bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7087 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7088 int64_t ImmOff =
C->getSExtValue();
7089 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7090 Base =
N.getOperand(0);
7091 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7098 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
static constexpr unsigned MaxRecursionDepth
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.