22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
47 AArch64DAGToDAGISel() =
delete;
64 std::vector<SDValue> &OutOps)
override;
66 template <
signed Low,
signed High,
signed Scale>
74 return SelectShiftedRegister(
N,
false, Reg, Shift);
77 return SelectShiftedRegister(
N,
true, Reg, Shift);
80 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
83 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
86 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
89 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
92 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
95 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
98 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
101 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
104 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
107 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
110 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
113 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
116 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
119 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
122 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
125 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
128 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
130 template <
unsigned Size,
unsigned Max>
134 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
136 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
137 int64_t
C = CI->getSExtValue();
145 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
152 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
158 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
163 N =
N->getOperand(0);
165 !isa<ConstantSDNode>(
N->getOperand(1)))
167 EVT VT =
N->getValueType(0);
168 EVT LVT =
N->getOperand(0).getValueType();
169 unsigned Index =
N->getConstantOperandVal(1);
173 Res =
N->getOperand(0);
181 EVT VT =
Op.getValueType();
182 unsigned ShtAmt =
N->getConstantOperandVal(1);
189 Op.getOperand(1).getConstantOperandVal(0)
190 <<
Op.getOperand(1).getConstantOperandVal(1));
192 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
194 Op.getOperand(1).getConstantOperandVal(0));
198 if (Imm != 1ULL << (ShtAmt - 1))
201 Res1 =
Op.getOperand(0);
202 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
206 bool SelectDupZeroOrUndef(
SDValue N) {
207 switch(
N->getOpcode()) {
212 auto Opnd0 =
N->getOperand(0);
227 switch(
N->getOpcode()) {
230 auto Opnd0 =
N->getOperand(0);
242 bool SelectDupNegativeZero(
SDValue N) {
243 switch(
N->getOpcode()) {
247 return Const && Const->isZero() && Const->isNegative();
254 template<MVT::SimpleValueType VT>
256 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
259 template <MVT::SimpleValueType VT,
bool Negate>
261 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
264 template <MVT::SimpleValueType VT>
266 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
269 template <MVT::SimpleValueType VT,
bool Invert = false>
271 return SelectSVELogicalImm(
N, VT, Imm, Invert);
274 template <MVT::SimpleValueType VT>
276 return SelectSVEArithImm(
N, VT, Imm);
279 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
281 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
288 EVT EltVT =
N->getValueType(0).getVectorElementType();
289 return SelectSVEShiftImm(
N->getOperand(0), 1,
295 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
297 if (!isa<ConstantSDNode>(
N))
300 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
302 MulImm = 1LL << MulImm;
304 if ((MulImm % std::abs(Scale)) != 0)
308 if ((MulImm >= Min) && (MulImm <= Max)) {
309 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
316 template <
signed Max,
signed Scale>
318 if (!isa<ConstantSDNode>(
N))
321 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
323 if (MulImm >= 0 && MulImm <= Max) {
325 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
332 template <
unsigned BaseReg,
unsigned Max>
334 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
340 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
363 const unsigned SubRegs[]);
365 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
367 bool tryIndexedLoad(
SDNode *
N);
369 bool trySelectStackSlotTagP(
SDNode *
N);
372 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
374 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
376 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
377 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
378 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
379 unsigned Opc_rr,
unsigned Opc_ri,
380 bool IsIntr =
false);
381 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
382 unsigned Scale,
unsigned Opc_ri,
384 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
385 bool IsZmMulti,
unsigned Opcode,
386 bool HasPred =
false);
387 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
388 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
389 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
390 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
391 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
392 bool IsTupleInput,
unsigned Opc);
393 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
395 template <
unsigned MaxIdx,
unsigned Scale>
396 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
398 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
unsigned Op,
399 unsigned MaxIdx,
unsigned Scale);
402 template <
int64_t Min,
int64_t Max>
406 template <
unsigned Scale>
408 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
411 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
414 template <
unsigned MaxIdx,
unsigned Scale>
419 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
420 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
421 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
422 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
423 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
424 unsigned Opc_rr,
unsigned Opc_ri);
425 std::tuple<unsigned, SDValue, SDValue>
426 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
430 bool tryBitfieldExtractOp(
SDNode *
N);
431 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
432 bool tryBitfieldInsertOp(
SDNode *
N);
433 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
434 bool tryShiftAmountMod(
SDNode *
N);
436 bool tryReadRegister(
SDNode *
N);
437 bool tryWriteRegister(
SDNode *
N);
439 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
440 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
445#include "AArch64GenDAGISel.inc"
453 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
455 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
468 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
469 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
470 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
473 template<
unsigned RegW
idth>
475 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
478 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
480 template<
unsigned RegW
idth>
482 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
488 bool SelectCMP_SWAP(
SDNode *
N);
498 bool AllowSaturation,
SDValue &Imm);
506 bool SelectAllActivePredicate(
SDValue N);
516 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
520char AArch64DAGToDAGISelLegacy::ID = 0;
528 Imm =
C->getZExtValue();
545 return N->getOpcode() == Opc &&
556 return Imm == ImmExpected;
560bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
562 std::vector<SDValue> &OutOps) {
563 switch(ConstraintID) {
566 case InlineAsm::ConstraintCode::m:
567 case InlineAsm::ConstraintCode::o:
568 case InlineAsm::ConstraintCode::Q:
574 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
576 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
577 dl,
Op.getValueType(),
579 OutOps.push_back(NewOp);
595 if (!isa<ConstantSDNode>(
N.getNode()))
598 uint64_t Immed =
N.getNode()->getAsZExtVal();
601 if (Immed >> 12 == 0) {
603 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
611 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
612 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
625 if (!isa<ConstantSDNode>(
N.getNode()))
629 uint64_t Immed =
N.getNode()->getAsZExtVal();
637 if (
N.getValueType() == MVT::i32)
640 Immed = ~Immed + 1ULL;
641 if (Immed & 0xFFFFFFFFFF000000ULL)
644 Immed &= 0xFFFFFFULL;
645 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
652 switch (
N.getOpcode()) {
671 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
674 unsigned ShiftVal = CSD->getZExtValue();
683 if (!isa<MemSDNode>(*UI))
685 if (!isa<MemSDNode>(*UII))
692bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
695 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
700 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
722bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
724 EVT VT =
N.getValueType();
725 if (VT != MVT::i32 && VT != MVT::i64)
728 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
734 unsigned LHSOpcode =
LHS->getOpcode();
748 unsigned LowZBits, MaskLen;
752 unsigned BitWidth =
N.getValueSizeInBits();
759 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
762 NewShiftC = LowZBits - ShiftAmtC;
763 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
769 NewShiftC = LowZBits + ShiftAmtC;
782 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
784 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
788 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
790 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
791 NewShiftAmt, BitWidthMinus1),
794 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
806 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
808 SrcVT =
N.getOperand(0).getValueType();
810 if (!IsLoadStore && SrcVT == MVT::i8)
812 else if (!IsLoadStore && SrcVT == MVT::i16)
814 else if (SrcVT == MVT::i32)
816 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
821 EVT SrcVT =
N.getOperand(0).getValueType();
822 if (!IsLoadStore && SrcVT == MVT::i8)
824 else if (!IsLoadStore && SrcVT == MVT::i16)
826 else if (SrcVT == MVT::i32)
828 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
856bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
859 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
864 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
865 V.getConstantOperandVal(1) <= 4 &&
878bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
880 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
890 unsigned BitSize =
N.getValueSizeInBits();
891 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
894 Reg =
N.getOperand(0);
895 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
896 return isWorthFoldingALU(
N,
true);
907 if (
N.getValueType() == MVT::i32)
915template<
signed Low,
signed High,
signed Scale>
917 if (!isa<ConstantSDNode>(
N))
920 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
921 if ((MulImm % std::abs(Scale)) == 0) {
922 int64_t RDVLImm = MulImm / Scale;
923 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
924 Imm = CurDAG->getTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
934bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
936 unsigned ShiftVal = 0;
951 Reg =
N.getOperand(0).getOperand(0);
957 Reg =
N.getOperand(0);
962 unsigned Opc =
N.getOpcode();
963 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
980 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
982 return isWorthFoldingALU(
N);
987bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
989 unsigned ShiftVal = 0;
1003 Reg =
N.getOperand(0);
1004 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1006 return isWorthFoldingALU(
N);
1015 for (
auto *
Use :
N->uses()) {
1042bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1043 unsigned BW,
unsigned Size,
1050 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1052 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1058 if (CurDAG->isBaseWithConstantOffset(
N)) {
1059 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1061 int64_t RHSC =
RHS->getSExtValue();
1063 int64_t
Range = 0x1LL << (BW - 1);
1065 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1066 RHSC < (
Range << Scale)) {
1067 Base =
N.getOperand(0);
1069 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1072 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1081 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1082 Base =
N.getOperand(0);
1084 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1087 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1098 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1105bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1111 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1113 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1119 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1120 Base =
N.getOperand(0);
1121 OffImm =
N.getOperand(1);
1130 if (CurDAG->isBaseWithConstantOffset(
N)) {
1131 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1132 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1135 Base =
N.getOperand(0);
1137 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1140 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1148 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1156 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1165bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1168 if (!CurDAG->isBaseWithConstantOffset(
N))
1170 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1171 int64_t RHSC =
RHS->getSExtValue();
1172 if (RHSC >= -256 && RHSC < 256) {
1173 Base =
N.getOperand(0);
1175 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1177 Base = CurDAG->getTargetFrameIndex(
1180 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1190 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1197bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1217 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1223 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1226 return isWorthFoldingAddr(
N,
Size);
1229bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1241 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1249 if (!isa<MemSDNode>(*UI))
1254 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1257 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1258 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1260 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1265 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1266 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1268 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1273 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1277 if (IsExtendedRegisterWorthFolding &&
1284 if (isWorthFoldingAddr(LHS,
Size))
1289 if (IsExtendedRegisterWorthFolding &&
1296 if (isWorthFoldingAddr(RHS,
Size))
1308 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1311 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1313 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1314 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1318bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1333 if (!isa<MemSDNode>(*UI))
1348 if (isa<ConstantSDNode>(RHS)) {
1349 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1359 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1362 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1366 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1369 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1370 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1372 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1377 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1378 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1380 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1387 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1388 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1394 static const unsigned RegClassIDs[] = {
1395 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1396 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1397 AArch64::dsub2, AArch64::dsub3};
1403 static const unsigned RegClassIDs[] = {
1404 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1405 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1406 AArch64::qsub2, AArch64::qsub3};
1412 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1413 AArch64::ZPR3RegClassID,
1414 AArch64::ZPR4RegClassID};
1415 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1416 AArch64::zsub2, AArch64::zsub3};
1426 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1427 AArch64::ZPR4Mul4RegClassID};
1428 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1429 AArch64::zsub2, AArch64::zsub3};
1434 const unsigned RegClassIDs[],
1435 const unsigned SubRegs[]) {
1438 if (Regs.
size() == 1)
1449 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1452 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1454 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1458 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1462void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1465 EVT VT =
N->getValueType(0);
1467 unsigned ExtOff = isExt;
1470 unsigned Vec0Off = ExtOff + 1;
1472 N->op_begin() + Vec0Off + NumVecs);
1479 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1480 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1483bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1485 if (
LD->isUnindexed())
1487 EVT VT =
LD->getMemoryVT();
1488 EVT DstVT =
N->getValueType(0);
1495 unsigned Opcode = 0;
1498 bool InsertTo64 =
false;
1500 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1501 else if (VT == MVT::i32) {
1503 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1505 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1507 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1513 }
else if (VT == MVT::i16) {
1515 if (DstVT == MVT::i64)
1516 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1518 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1520 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1521 InsertTo64 = DstVT == MVT::i64;
1526 }
else if (VT == MVT::i8) {
1528 if (DstVT == MVT::i64)
1529 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1531 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1533 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1534 InsertTo64 = DstVT == MVT::i64;
1539 }
else if (VT == MVT::f16) {
1540 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1541 }
else if (VT == MVT::bf16) {
1542 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1543 }
else if (VT == MVT::f32) {
1544 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1546 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1548 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1556 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1558 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1563 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1568 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1570 SDValue(CurDAG->getMachineNode(
1571 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1572 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1577 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1580 CurDAG->RemoveDeadNode(
N);
1584void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1585 unsigned SubRegIdx) {
1587 EVT VT =
N->getValueType(0);
1593 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1595 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1597 for (
unsigned i = 0; i < NumVecs; ++i)
1599 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1605 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1607 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1610 CurDAG->RemoveDeadNode(
N);
1613void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1614 unsigned Opc,
unsigned SubRegIdx) {
1616 EVT VT =
N->getValueType(0);
1623 const EVT ResTys[] = {MVT::i64,
1624 MVT::Untyped, MVT::Other};
1626 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1634 ReplaceUses(
SDValue(
N, 0), SuperReg);
1636 for (
unsigned i = 0; i < NumVecs; ++i)
1638 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1642 CurDAG->RemoveDeadNode(
N);
1648std::tuple<unsigned, SDValue, SDValue>
1649AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1655 SDValue NewOffset = OldOffset;
1657 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1658 N, OldBase, NewBase, NewOffset);
1662 const bool IsRegReg =
1663 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1666 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1679template <SelectTypeKind Kind>
1691 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1696 if (EltVT != MVT::i1)
1700 if (EltVT == MVT::bf16)
1702 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1732void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1735 if (
Imm->getZExtValue() > 1)
1739 EVT VT =
N->getValueType(0);
1740 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1741 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1744 for (
unsigned I = 0;
I < 2; ++
I)
1745 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1746 AArch64::psub0 +
I,
DL, VT, SuperReg));
1748 CurDAG->RemoveDeadNode(
N);
1751void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1753 EVT VT =
N->getValueType(0);
1755 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1757 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1760 for (
unsigned I = 0;
I < 2; ++
I)
1761 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1762 AArch64::psub0 +
I,
DL, VT, SuperReg));
1764 CurDAG->RemoveDeadNode(
N);
1767void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1769 EVT VT =
N->getValueType(0);
1771 SDValue Ops = createZTuple(Regs);
1775 for (
unsigned i = 0; i < NumVecs; ++i)
1776 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1777 AArch64::zsub0 + i,
DL, VT, SuperReg));
1779 CurDAG->RemoveDeadNode(
N);
1782void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1787 assert(Opcode != 0 &&
"Unexpected opcode");
1790 EVT VT =
N->getValueType(0);
1791 unsigned FirstVecIdx = HasPred ? 2 : 1;
1793 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1795 N->op_begin() + StartIdx + NumVecs);
1796 return createZMulTuple(Regs);
1799 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1803 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1805 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1809 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1810 N->getOperand(1), Zdn, Zm);
1812 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1814 for (
unsigned i = 0; i < NumVecs; ++i)
1815 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1816 AArch64::zsub0 + i,
DL, VT, SuperReg));
1818 CurDAG->RemoveDeadNode(
N);
1821void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1822 unsigned Scale,
unsigned Opc_ri,
1823 unsigned Opc_rr,
bool IsIntr) {
1824 assert(Scale < 5 &&
"Invalid scaling value.");
1826 EVT VT =
N->getValueType(0);
1832 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1833 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1834 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1836 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1840 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1842 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1844 for (
unsigned i = 0; i < NumVecs; ++i)
1845 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1846 AArch64::zsub0 + i,
DL, VT, SuperReg));
1849 unsigned ChainIdx = NumVecs;
1851 CurDAG->RemoveDeadNode(
N);
1854void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1859 assert(Scale < 4 &&
"Invalid scaling value.");
1861 EVT VT =
N->getValueType(0);
1869 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1875 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1877 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1879 for (
unsigned i = 0; i < NumVecs; ++i)
1880 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1881 AArch64::zsub0 + i,
DL, VT, SuperReg));
1884 unsigned ChainIdx = NumVecs;
1886 CurDAG->RemoveDeadNode(
N);
1889void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1891 if (
N->getValueType(0) != MVT::nxv4f32)
1893 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1896void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
1897 unsigned NumOutVecs,
1900 if (
Imm->getZExtValue() > MaxImm)
1904 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
1908 EVT VT =
Node->getValueType(0);
1911 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
1914 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
1915 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
1916 AArch64::zsub0 +
I,
DL, VT, SuperReg));
1919 unsigned ChainIdx = NumOutVecs;
1921 CurDAG->RemoveDeadNode(
Node);
1924void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
1927 EVT VT =
N->getValueType(0);
1930 SDValue Zd = createZMulTuple(Regs);
1931 SDValue Zn =
N->getOperand(1 + NumVecs);
1932 SDValue Zm =
N->getOperand(2 + NumVecs);
1938 for (
unsigned i = 0; i < NumVecs; ++i)
1939 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1940 AArch64::zsub0 + i,
DL, VT, SuperReg));
1942 CurDAG->RemoveDeadNode(
N);
1972template <
unsigned MaxIdx,
unsigned Scale>
1973void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
1974 unsigned BaseReg,
unsigned Op) {
1975 unsigned TileNum = 0;
1976 if (BaseReg != AArch64::ZA)
1977 TileNum =
N->getConstantOperandVal(2);
1983 if (BaseReg == AArch64::ZA)
1984 SliceBase =
N->getOperand(2);
1986 SliceBase =
N->getOperand(3);
1988 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
1994 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
1996 EVT VT =
N->getValueType(0);
1997 for (
unsigned I = 0;
I < NumVecs; ++
I)
1999 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2002 unsigned ChainIdx = NumVecs;
2004 CurDAG->RemoveDeadNode(
N);
2007void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
2008 unsigned Op,
unsigned MaxIdx,
2011 SDValue SliceBase =
N->getOperand(3);
2013 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2021 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2023 EVT VT =
N->getValueType(0);
2024 for (
unsigned I = 0;
I < NumVecs; ++
I)
2026 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2030 unsigned ChainIdx = NumVecs;
2032 CurDAG->RemoveDeadNode(
N);
2035void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2036 unsigned NumOutVecs,
2040 EVT VT =
N->getValueType(0);
2041 unsigned NumInVecs =
N->getNumOperands() - 1;
2045 assert((NumInVecs == 2 || NumInVecs == 4) &&
2046 "Don't know how to handle multi-register input!");
2048 N->op_begin() + 1 + NumInVecs);
2052 for (
unsigned I = 0;
I < NumInVecs;
I++)
2056 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2059 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2060 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2061 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2062 CurDAG->RemoveDeadNode(
N);
2065void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2068 EVT VT =
N->getOperand(2)->getValueType(0);
2075 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2076 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2080 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2085void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2086 unsigned Scale,
unsigned Opc_rr,
2092 SDValue RegSeq = createZTuple(Regs);
2097 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2098 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2099 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2101 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2105 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2117 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2118 int FI = FINode->getIndex();
2120 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2127void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2130 EVT VT =
N->getOperand(2)->getValueType(0);
2131 const EVT ResTys[] = {MVT::i64,
2140 N->getOperand(NumVecs + 1),
2141 N->getOperand(NumVecs + 2),
2143 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2183void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2186 EVT VT =
N->getValueType(0);
2194 WidenVector(*CurDAG));
2198 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2200 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2202 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2203 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2204 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2208 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2209 AArch64::qsub2, AArch64::qsub3 };
2210 for (
unsigned i = 0; i < NumVecs; ++i) {
2211 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2218 CurDAG->RemoveDeadNode(
N);
2221void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2224 EVT VT =
N->getValueType(0);
2232 WidenVector(*CurDAG));
2236 const EVT ResTys[] = {MVT::i64,
2239 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2242 CurDAG->getTargetConstant(LaneNo, dl,
2244 N->getOperand(NumVecs + 2),
2245 N->getOperand(NumVecs + 3),
2247 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2259 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2260 AArch64::qsub2, AArch64::qsub3 };
2261 for (
unsigned i = 0; i < NumVecs; ++i) {
2262 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2272 CurDAG->RemoveDeadNode(
N);
2275void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2278 EVT VT =
N->getOperand(2)->getValueType(0);
2286 WidenVector(*CurDAG));
2290 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2292 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2293 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2294 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2298 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2303void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2306 EVT VT =
N->getOperand(2)->getValueType(0);
2314 WidenVector(*CurDAG));
2318 const EVT ResTys[] = {MVT::i64,
2321 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2323 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2324 N->getOperand(NumVecs + 2),
2325 N->getOperand(NumVecs + 3),
2327 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2331 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2338 unsigned &LSB,
unsigned &MSB,
2339 unsigned NumberOfIgnoredLowBits,
2340 bool BiggerPattern) {
2342 "N must be a AND operation to call this function");
2344 EVT VT =
N->getValueType(0);
2349 assert((VT == MVT::i32 || VT == MVT::i64) &&
2350 "Type checking must have been done before calling this function");
2364 const SDNode *Op0 =
N->getOperand(0).getNode();
2368 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2371 if (AndImm & (AndImm + 1))
2374 bool ClampMSB =
false;
2394 ClampMSB = (VT == MVT::i32);
2395 }
else if (BiggerPattern) {
2401 Opd0 =
N->getOperand(0);
2407 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2410 <<
": Found large shift immediate, this should not happen\n"));
2416 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2417 : llvm::countr_one<uint64_t>(AndImm)) -
2424 MSB = MSB > 31 ? 31 : MSB;
2426 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2431 SDValue &Opd0,
unsigned &Immr,
2435 EVT VT =
N->getValueType(0);
2437 assert((VT == MVT::i32 || VT == MVT::i64) &&
2438 "Type checking must have been done before calling this function");
2442 Op =
Op->getOperand(0);
2443 VT =
Op->getValueType(0);
2452 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2456 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2457 Opd0 =
Op.getOperand(0);
2459 Imms = ShiftImm + Width - 1;
2487 Opd0 =
N->getOperand(0).getOperand(0);
2497 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2504 unsigned &Immr,
unsigned &Imms,
2505 bool BiggerPattern) {
2507 "N must be a SHR/SRA operation to call this function");
2509 EVT VT =
N->getValueType(0);
2514 assert((VT == MVT::i32 || VT == MVT::i64) &&
2515 "Type checking must have been done before calling this function");
2525 Opd0 =
N->getOperand(0).getOperand(0);
2526 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2532 Opd0 =
N->getOperand(0).getOperand(0);
2535 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2536 }
else if (BiggerPattern) {
2540 Opd0 =
N->getOperand(0);
2549 <<
": Found large shift immediate, this should not happen\n"));
2558 "bad amount in shift node!");
2559 int immr = SrlImm - ShlImm;
2564 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2566 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2570bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2573 EVT VT =
N->getValueType(0);
2574 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2575 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2586 unsigned Immr = ShiftImm;
2588 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2589 CurDAG->getTargetConstant(Imms, dl, VT)};
2590 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2595 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2596 unsigned NumberOfIgnoredLowBits = 0,
2597 bool BiggerPattern =
false) {
2598 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2601 switch (
N->getOpcode()) {
2603 if (!
N->isMachineOpcode())
2608 NumberOfIgnoredLowBits, BiggerPattern);
2617 unsigned NOpc =
N->getMachineOpcode();
2621 case AArch64::SBFMWri:
2622 case AArch64::UBFMWri:
2623 case AArch64::SBFMXri:
2624 case AArch64::UBFMXri:
2626 Opd0 =
N->getOperand(0);
2627 Immr =
N->getConstantOperandVal(1);
2628 Imms =
N->getConstantOperandVal(2);
2635bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2636 unsigned Opc, Immr, Imms;
2641 EVT VT =
N->getValueType(0);
2646 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2647 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2648 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2650 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2651 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2657 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2658 CurDAG->getTargetConstant(Imms, dl, VT)};
2659 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2668 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2669 assert((VT == MVT::i32 || VT == MVT::i64) &&
2670 "i32 or i64 mask type expected!");
2676 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2677 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2700 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2710 APInt OpUsefulBits(UsefulBits);
2714 OpUsefulBits <<= MSB - Imm + 1;
2719 OpUsefulBits <<= Imm;
2721 OpUsefulBits <<= MSB + 1;
2724 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2730 UsefulBits &= OpUsefulBits;
2736 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2738 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2746 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2747 APInt Mask(UsefulBits);
2748 Mask.clearAllBits();
2756 Mask.lshrInPlace(ShiftAmt);
2762 Mask.lshrInPlace(ShiftAmt);
2774 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2776 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2778 APInt OpUsefulBits(UsefulBits);
2792 OpUsefulBits <<= Width;
2795 if (
Op.getOperand(1) == Orig) {
2797 Mask = ResultUsefulBits & OpUsefulBits;
2801 if (
Op.getOperand(0) == Orig)
2803 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2809 OpUsefulBits <<= Width;
2811 OpUsefulBits <<= LSB;
2813 if (
Op.getOperand(1) == Orig) {
2815 Mask = ResultUsefulBits & OpUsefulBits;
2816 Mask.lshrInPlace(LSB);
2819 if (
Op.getOperand(0) == Orig)
2820 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2837 case AArch64::ANDSWri:
2838 case AArch64::ANDSXri:
2839 case AArch64::ANDWri:
2840 case AArch64::ANDXri:
2844 case AArch64::UBFMWri:
2845 case AArch64::UBFMXri:
2848 case AArch64::ORRWrs:
2849 case AArch64::ORRXrs:
2854 case AArch64::BFMWri:
2855 case AArch64::BFMXri:
2858 case AArch64::STRBBui:
2859 case AArch64::STURBBi:
2865 case AArch64::STRHHui:
2866 case AArch64::STURHHi:
2879 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
2881 UsefulBits =
APInt(Bitwidth, 0);
2890 UsersUsefulBits |= UsefulBitsForUse;
2895 UsefulBits &= UsersUsefulBits;
2905 EVT VT =
Op.getValueType();
2908 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2911 if (ShlAmount > 0) {
2914 UBFMOpc, dl, VT,
Op,
2919 assert(ShlAmount < 0 &&
"expected right shift");
2920 int ShrAmount = -ShlAmount;
2946 bool BiggerPattern,
SDValue &Src,
2947 int &DstLSB,
int &Width) {
2948 EVT VT =
Op.getValueType();
2957 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2961 switch (
Op.getOpcode()) {
2966 NonZeroBits, Src, DstLSB, Width);
2969 NonZeroBits, Src, DstLSB, Width);
2982 EVT VT =
Op.getValueType();
2983 assert((VT == MVT::i32 || VT == MVT::i64) &&
2984 "Caller guarantees VT is one of i32 or i64");
2997 assert((~AndImm & NonZeroBits) == 0 &&
2998 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3027 if (!BiggerPattern && !AndOp0.
hasOneUse())
3046 <<
"Found large Width in bit-field-positioning -- this indicates no "
3047 "proper combining / constant folding was performed\n");
3056 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3071 "Op.getNode() should be a SHL node to call this function");
3073 "Op.getNode() should shift ShlImm to call this function");
3080 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3104 EVT VT =
Op.getValueType();
3105 assert((VT == MVT::i32 || VT == MVT::i64) &&
3106 "Caller guarantees that type is i32 or i64");
3113 if (!BiggerPattern && !
Op.hasOneUse())
3122 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3130 assert(VT == MVT::i32 || VT == MVT::i64);
3141 EVT VT =
N->getValueType(0);
3142 if (VT != MVT::i32 && VT != MVT::i64)
3160 if (!
And.hasOneUse() ||
3170 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3177 if ((OrImm & NotKnownZero) != 0) {
3189 unsigned ImmS = Width - 1;
3195 bool IsBFI = LSB != 0;
3200 unsigned OrChunks = 0, BFIChunks = 0;
3201 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3202 if (((OrImm >> Shift) & 0xFFFF) != 0)
3204 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3207 if (BFIChunks > OrChunks)
3213 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3221 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3230 if (!Dst.hasOneUse())
3233 EVT VT = Dst.getValueType();
3234 assert((VT == MVT::i32 || VT == MVT::i64) &&
3235 "Caller should guarantee that VT is one of i32 or i64");
3243 SDValue DstOp0 = Dst.getOperand(0);
3263 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3264 unsigned MaskWidth =
3267 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3273 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3274 ShiftedOperand =
SDValue(UBFMNode, 0);
3284 ShiftedOperand = Dst.getOperand(0);
3291 ShiftedOperand = Dst.getOperand(0);
3303 const bool BiggerPattern) {
3304 EVT VT =
N->getValueType(0);
3305 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3306 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3307 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3308 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3309 assert((VT == MVT::i32 || VT == MVT::i64) &&
3310 "Expect result type to be i32 or i64 since N is combinable to BFM");
3317 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3320 if (BiggerPattern) {
3334 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3343 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3405 EVT VT =
N->getValueType(0);
3406 if (VT != MVT::i32 && VT != MVT::i64)
3414 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3415 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3435 for (
int I = 0;
I < 4; ++
I) {
3438 unsigned ImmR, ImmS;
3439 bool BiggerPattern =
I / 2;
3440 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3442 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3448 NumberOfIgnoredLowBits, BiggerPattern)) {
3451 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3452 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3457 Width = ImmS - ImmR + 1;
3468 Src, DstLSB, Width)) {
3476 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3486 APInt BitsToBeInserted =
3489 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3513 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3546 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3548 if (Src->hasOneUse() &&
3551 Src = Src->getOperand(0);
3561 unsigned ImmS = Width - 1;
3567 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3575bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3584 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3597bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3601 EVT VT =
N->getValueType(0);
3602 if (VT != MVT::i32 && VT != MVT::i64)
3608 Op0, DstLSB, Width))
3614 unsigned ImmS = Width - 1;
3617 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3618 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3619 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3620 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3626bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3627 EVT VT =
N->getValueType(0);
3630 switch (
N->getOpcode()) {
3632 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3635 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3638 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3641 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3649 if (VT == MVT::i32) {
3652 }
else if (VT == MVT::i64) {
3658 SDValue ShiftAmt =
N->getOperand(1);
3678 (Add0Imm %
Size == 0)) {
3684 if (SubVT == MVT::i32) {
3685 NegOpc = AArch64::SUBWrr;
3686 ZeroReg = AArch64::WZR;
3688 assert(SubVT == MVT::i64);
3689 NegOpc = AArch64::SUBXrr;
3690 ZeroReg = AArch64::XZR;
3693 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3695 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3696 NewShiftAmt =
SDValue(Neg, 0);
3704 if (SubVT == MVT::i32) {
3705 NotOpc = AArch64::ORNWrr;
3706 ZeroReg = AArch64::WZR;
3708 assert(SubVT == MVT::i64);
3709 NotOpc = AArch64::ORNXrr;
3710 ZeroReg = AArch64::XZR;
3713 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3715 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3716 NewShiftAmt =
SDValue(Not, 0);
3737 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3738 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3740 AArch64::SUBREG_TO_REG,
DL, VT,
3741 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3742 NewShiftAmt =
SDValue(Ext, 0);
3745 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3746 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3753 bool isReciprocal) {
3756 FVal = CN->getValueAPF();
3757 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3760 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3764 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3765 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3788 if (!IsExact || !IntVal.isPowerOf2())
3790 unsigned FBits = IntVal.logBase2();
3794 if (FBits == 0 || FBits > RegWidth)
return false;
3800bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3801 unsigned RegWidth) {
3806bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3808 unsigned RegWidth) {
3818 RegString.
split(Fields,
':');
3820 if (Fields.
size() == 1)
3824 &&
"Invalid number of fields in read register string");
3827 bool AllIntFields =
true;
3831 AllIntFields &= !
Field.getAsInteger(10, IntField);
3836 "Unexpected non-integer value in special register string.");
3841 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3842 (Ops[3] << 3) | (Ops[4]);
3849bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3850 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3851 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3856 unsigned Opcode64Bit = AArch64::MRS;
3861 const auto *TheReg =
3863 if (TheReg && TheReg->Readable &&
3864 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3865 Imm = TheReg->Encoding;
3871 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3872 Opcode64Bit = AArch64::ADR;
3880 SDValue InChain =
N->getOperand(0);
3881 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
3882 if (!ReadIs128Bit) {
3883 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
3884 {SysRegImm, InChain});
3888 {MVT::Untyped , MVT::Other },
3889 {SysRegImm, InChain});
3893 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
3895 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
3901 ReplaceUses(
SDValue(
N, 2), OutChain);
3910bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
3911 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3912 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3917 if (!WriteIs128Bit) {
3923 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
3925 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
3926 "Expected a constant integer expression.");
3927 unsigned Reg = PMapper->Encoding;
3928 uint64_t Immed =
N->getConstantOperandVal(2);
3929 CurDAG->SelectNodeTo(
3930 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
3931 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
3937 if (trySelectPState(
3938 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
3939 AArch64::MSRpstateImm4))
3941 if (trySelectPState(
3942 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
3943 AArch64::MSRpstateImm1))
3953 if (TheReg && TheReg->Writeable &&
3954 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3955 Imm = TheReg->Encoding;
3963 SDValue InChain =
N->getOperand(0);
3964 if (!WriteIs128Bit) {
3965 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
3966 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3967 N->getOperand(2), InChain);
3971 SDNode *Pair = CurDAG->getMachineNode(
3972 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
3973 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
3976 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
3978 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
3980 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
3981 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3989bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3991 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3994 if (Subtarget->hasLSE())
return false;
3996 if (MemTy == MVT::i8)
3997 Opcode = AArch64::CMP_SWAP_8;
3998 else if (MemTy == MVT::i16)
3999 Opcode = AArch64::CMP_SWAP_16;
4000 else if (MemTy == MVT::i32)
4001 Opcode = AArch64::CMP_SWAP_32;
4002 else if (MemTy == MVT::i64)
4003 Opcode = AArch64::CMP_SWAP_64;
4007 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4008 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4010 SDNode *CmpSwap = CurDAG->getMachineNode(
4012 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4015 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
4019 CurDAG->RemoveDeadNode(
N);
4026 if (!isa<ConstantSDNode>(
N))
4038 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4039 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4046 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4047 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4051 if (Val <= 65280 && Val % 256 == 0) {
4052 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4053 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4064bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4067 if (!isa<ConstantSDNode>(
N))
4071 int64_t Val = cast<ConstantSDNode>(
N)
4088 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4089 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4096 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4097 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4101 if (Val <= 65280 && Val % 256 == 0) {
4102 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4103 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4116 if (!isa<ConstantSDNode>(
N))
4120 int64_t Val = cast<ConstantSDNode>(
N)
4128 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4129 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4135 if (Val >= -128 && Val <= 127) {
4136 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4137 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4141 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4142 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4143 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4154bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4155 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4156 int64_t ImmVal = CNode->getSExtValue();
4158 if (ImmVal >= -128 && ImmVal < 128) {
4159 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4167 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4168 uint64_t ImmVal = CNode->getZExtValue();
4178 ImmVal &= 0xFFFFFFFF;
4187 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4196 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4197 uint64_t ImmVal = CNode->getZExtValue();
4207 ImmVal |= ImmVal << 8;
4208 ImmVal |= ImmVal << 16;
4209 ImmVal |= ImmVal << 32;
4213 ImmVal |= ImmVal << 16;
4214 ImmVal |= ImmVal << 32;
4217 ImmVal &= 0xFFFFFFFF;
4218 ImmVal |= ImmVal << 32;
4228 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4243 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4244 uint64_t ImmVal = CN->getZExtValue();
4251 if (ImmVal >
High) {
4252 if (!AllowSaturation)
4257 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4264bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4268 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4280 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4281 SDValue FiOp = CurDAG->getTargetFrameIndex(
4283 int TagOffset =
N->getConstantOperandVal(3);
4285 SDNode *Out = CurDAG->getMachineNode(
4286 AArch64::TAGPstack,
DL, MVT::i64,
4287 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4288 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4289 ReplaceNode(
N, Out);
4293void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4294 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4295 "llvm.aarch64.tagp third argument must be an immediate");
4296 if (trySelectStackSlotTagP(
N))
4303 int TagOffset =
N->getConstantOperandVal(3);
4304 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4305 {
N->getOperand(1),
N->getOperand(2)});
4306 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4307 {
SDValue(N1, 0),
N->getOperand(2)});
4308 SDNode *N3 = CurDAG->getMachineNode(
4309 AArch64::ADDG,
DL, MVT::i64,
4310 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4311 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4315bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4319 if (
N->getConstantOperandVal(2) != 0)
4321 if (!
N->getOperand(0).isUndef())
4325 EVT VT =
N->getValueType(0);
4326 EVT InVT =
N->getOperand(1).getValueType();
4337 "Expected to insert into a packed scalable vector!");
4340 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4341 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4342 N->getOperand(1), RC));
4346bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4350 if (
N->getConstantOperandVal(1) != 0)
4354 EVT VT =
N->getValueType(0);
4355 EVT InVT =
N->getOperand(0).getValueType();
4366 "Expected to extract from a packed scalable vector!");
4369 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4370 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4371 N->getOperand(0), RC));
4375bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4380 EVT VT =
N->getValueType(0);
4392 (Subtarget->hasSVE2() ||
4393 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4402 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4403 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4410 APInt ShlAmt, ShrAmt;
4423 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4424 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4425 AArch64::XAR_ZZZI_D})) {
4426 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4432 if (!Subtarget->hasSHA3())
4454 if (ShAmt + HsAmt != 64)
4458 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4463void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4465 if (
Node->isMachineOpcode()) {
4467 Node->setNodeId(-1);
4472 EVT VT =
Node->getValueType(0);
4474 switch (
Node->getOpcode()) {
4479 if (SelectCMP_SWAP(
Node))
4485 if (tryReadRegister(
Node))
4491 if (tryWriteRegister(
Node))
4498 if (tryIndexedLoad(
Node))
4507 if (tryBitfieldExtractOp(
Node))
4509 if (tryBitfieldInsertInZeroOp(
Node))
4514 if (tryShiftAmountMod(
Node))
4519 if (tryBitfieldExtractOpFromSExt(
Node))
4524 if (tryBitfieldInsertOp(
Node))
4526 if (trySelectXAR(
Node))
4531 if (trySelectCastScalableToFixedLengthVector(
Node))
4537 if (trySelectCastFixedLengthToScalableVector(
Node))
4546 if (ConstNode->
isZero()) {
4547 if (VT == MVT::i32) {
4549 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4550 ReplaceNode(
Node,
New.getNode());
4552 }
else if (VT == MVT::i64) {
4554 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4555 ReplaceNode(
Node,
New.getNode());
4564 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4567 SDValue TFI = CurDAG->getTargetFrameIndex(
4570 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4571 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4572 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4576 unsigned IntNo =
Node->getConstantOperandVal(1);
4580 case Intrinsic::aarch64_ldaxp:
4581 case Intrinsic::aarch64_ldxp: {
4583 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4588 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4589 MVT::Other, MemAddr, Chain);
4593 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4594 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4595 ReplaceNode(
Node, Ld);
4598 case Intrinsic::aarch64_stlxp:
4599 case Intrinsic::aarch64_stxp: {
4601 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4609 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4611 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4614 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4615 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4617 ReplaceNode(
Node, St);
4620 case Intrinsic::aarch64_neon_ld1x2:
4621 if (VT == MVT::v8i8) {
4622 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4624 }
else if (VT == MVT::v16i8) {
4625 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4627 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4628 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4630 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4631 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4633 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4634 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4636 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4637 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4639 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4640 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4642 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4643 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4647 case Intrinsic::aarch64_neon_ld1x3:
4648 if (VT == MVT::v8i8) {
4649 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4651 }
else if (VT == MVT::v16i8) {
4652 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4654 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4655 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4657 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4658 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4660 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4661 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4663 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4664 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4666 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4667 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4669 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4670 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4674 case Intrinsic::aarch64_neon_ld1x4:
4675 if (VT == MVT::v8i8) {
4676 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4678 }
else if (VT == MVT::v16i8) {
4679 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4681 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4682 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4684 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4685 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4687 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4688 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4690 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4691 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4693 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4694 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4696 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4697 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4701 case Intrinsic::aarch64_neon_ld2:
4702 if (VT == MVT::v8i8) {
4703 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4705 }
else if (VT == MVT::v16i8) {
4706 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4708 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4709 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4711 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4712 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4714 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4715 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4717 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4718 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4720 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4721 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4723 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4724 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4728 case Intrinsic::aarch64_neon_ld3:
4729 if (VT == MVT::v8i8) {
4730 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4732 }
else if (VT == MVT::v16i8) {
4733 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4735 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4736 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4738 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4739 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4741 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4742 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4744 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4745 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4747 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4748 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4750 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4751 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4755 case Intrinsic::aarch64_neon_ld4:
4756 if (VT == MVT::v8i8) {
4757 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4759 }
else if (VT == MVT::v16i8) {
4760 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4762 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4763 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4765 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4766 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4768 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4769 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4771 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4772 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4774 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4775 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4777 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4778 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4782 case Intrinsic::aarch64_neon_ld2r:
4783 if (VT == MVT::v8i8) {
4784 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4786 }
else if (VT == MVT::v16i8) {
4787 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4789 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4790 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4792 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4793 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4795 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4796 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4798 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4799 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4801 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4802 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4804 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4805 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4809 case Intrinsic::aarch64_neon_ld3r:
4810 if (VT == MVT::v8i8) {
4811 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4813 }
else if (VT == MVT::v16i8) {
4814 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4816 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4817 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4819 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4820 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4822 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4823 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4825 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4826 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4828 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4829 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4831 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4832 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4836 case Intrinsic::aarch64_neon_ld4r:
4837 if (VT == MVT::v8i8) {
4838 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4840 }
else if (VT == MVT::v16i8) {
4841 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4843 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4844 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4846 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4847 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
4849 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4850 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
4852 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4853 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
4855 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4856 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
4858 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4859 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
4863 case Intrinsic::aarch64_neon_ld2lane:
4864 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4865 SelectLoadLane(
Node, 2, AArch64::LD2i8);
4867 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4868 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4869 SelectLoadLane(
Node, 2, AArch64::LD2i16);
4871 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4873 SelectLoadLane(
Node, 2, AArch64::LD2i32);
4875 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4877 SelectLoadLane(
Node, 2, AArch64::LD2i64);
4881 case Intrinsic::aarch64_neon_ld3lane:
4882 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4883 SelectLoadLane(
Node, 3, AArch64::LD3i8);
4885 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4886 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4887 SelectLoadLane(
Node, 3, AArch64::LD3i16);
4889 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4891 SelectLoadLane(
Node, 3, AArch64::LD3i32);
4893 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4895 SelectLoadLane(
Node, 3, AArch64::LD3i64);
4899 case Intrinsic::aarch64_neon_ld4lane:
4900 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4901 SelectLoadLane(
Node, 4, AArch64::LD4i8);
4903 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4904 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4905 SelectLoadLane(
Node, 4, AArch64::LD4i16);
4907 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4909 SelectLoadLane(
Node, 4, AArch64::LD4i32);
4911 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4913 SelectLoadLane(
Node, 4, AArch64::LD4i64);
4917 case Intrinsic::aarch64_ld64b:
4918 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
4920 case Intrinsic::aarch64_sve_ld2q_sret: {
4921 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
4924 case Intrinsic::aarch64_sve_ld3q_sret: {
4925 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
4928 case Intrinsic::aarch64_sve_ld4q_sret: {
4929 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
4932 case Intrinsic::aarch64_sve_ld2_sret: {
4933 if (VT == MVT::nxv16i8) {
4934 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
4937 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4938 VT == MVT::nxv8bf16) {
4939 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
4942 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4943 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
4946 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4947 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
4953 case Intrinsic::aarch64_sve_ld1_pn_x2: {
4954 if (VT == MVT::nxv16i8) {
4955 if (Subtarget->hasSME2())
4956 SelectContiguousMultiVectorLoad(
4957 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
4958 else if (Subtarget->hasSVE2p1())
4959 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
4964 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4965 VT == MVT::nxv8bf16) {
4966 if (Subtarget->hasSME2())
4967 SelectContiguousMultiVectorLoad(
4968 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
4969 else if (Subtarget->hasSVE2p1())
4970 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
4975 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4976 if (Subtarget->hasSME2())
4977 SelectContiguousMultiVectorLoad(
4978 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
4979 else if (Subtarget->hasSVE2p1())
4980 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
4985 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4986 if (Subtarget->hasSME2())
4987 SelectContiguousMultiVectorLoad(
4988 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
4989 else if (Subtarget->hasSVE2p1())
4990 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
4998 case Intrinsic::aarch64_sve_ld1_pn_x4: {
4999 if (VT == MVT::nxv16i8) {
5000 if (Subtarget->hasSME2())
5001 SelectContiguousMultiVectorLoad(
5002 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5003 else if (Subtarget->hasSVE2p1())
5004 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
5009 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5010 VT == MVT::nxv8bf16) {
5011 if (Subtarget->hasSME2())
5012 SelectContiguousMultiVectorLoad(
5013 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5014 else if (Subtarget->hasSVE2p1())
5015 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
5020 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5021 if (Subtarget->hasSME2())
5022 SelectContiguousMultiVectorLoad(
5023 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5024 else if (Subtarget->hasSVE2p1())
5025 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
5030 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5031 if (Subtarget->hasSME2())
5032 SelectContiguousMultiVectorLoad(
5033 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5034 else if (Subtarget->hasSVE2p1())
5035 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5043 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5044 if (VT == MVT::nxv16i8) {
5045 if (Subtarget->hasSME2())
5046 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5047 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5048 AArch64::LDNT1B_2Z_PSEUDO);
5049 else if (Subtarget->hasSVE2p1())
5050 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5051 AArch64::LDNT1B_2Z);
5055 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5056 VT == MVT::nxv8bf16) {
5057 if (Subtarget->hasSME2())
5058 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5059 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5060 AArch64::LDNT1H_2Z_PSEUDO);
5061 else if (Subtarget->hasSVE2p1())
5062 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5063 AArch64::LDNT1H_2Z);
5067 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5068 if (Subtarget->hasSME2())
5069 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5070 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5071 AArch64::LDNT1W_2Z_PSEUDO);
5072 else if (Subtarget->hasSVE2p1())
5073 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5074 AArch64::LDNT1W_2Z);
5078 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5079 if (Subtarget->hasSME2())
5080 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5081 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5082 AArch64::LDNT1D_2Z_PSEUDO);
5083 else if (Subtarget->hasSVE2p1())
5084 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5085 AArch64::LDNT1D_2Z);
5092 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5093 if (VT == MVT::nxv16i8) {
5094 if (Subtarget->hasSME2())
5095 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5096 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5097 AArch64::LDNT1B_4Z_PSEUDO);
5098 else if (Subtarget->hasSVE2p1())
5099 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5100 AArch64::LDNT1B_4Z);
5104 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5105 VT == MVT::nxv8bf16) {
5106 if (Subtarget->hasSME2())
5107 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5108 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5109 AArch64::LDNT1H_4Z_PSEUDO);
5110 else if (Subtarget->hasSVE2p1())
5111 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5112 AArch64::LDNT1H_4Z);
5116 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5117 if (Subtarget->hasSME2())
5118 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5119 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5120 AArch64::LDNT1W_4Z_PSEUDO);
5121 else if (Subtarget->hasSVE2p1())
5122 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5123 AArch64::LDNT1W_4Z);
5127 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5128 if (Subtarget->hasSME2())
5129 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5130 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5131 AArch64::LDNT1D_4Z_PSEUDO);
5132 else if (Subtarget->hasSVE2p1())
5133 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5134 AArch64::LDNT1D_4Z);
5141 case Intrinsic::aarch64_sve_ld3_sret: {
5142 if (VT == MVT::nxv16i8) {
5143 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5146 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5147 VT == MVT::nxv8bf16) {
5148 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5151 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5152 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5155 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5156 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5162 case Intrinsic::aarch64_sve_ld4_sret: {
5163 if (VT == MVT::nxv16i8) {
5164 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5167 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5168 VT == MVT::nxv8bf16) {
5169 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5172 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5173 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5176 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5177 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5183 case Intrinsic::aarch64_sme_read_hor_vg2: {
5184 if (VT == MVT::nxv16i8) {
5185 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5186 AArch64::MOVA_2ZMXI_H_B);
5188 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5189 VT == MVT::nxv8bf16) {
5190 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5191 AArch64::MOVA_2ZMXI_H_H);
5193 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5194 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5195 AArch64::MOVA_2ZMXI_H_S);
5197 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5198 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5199 AArch64::MOVA_2ZMXI_H_D);
5204 case Intrinsic::aarch64_sme_read_ver_vg2: {
5205 if (VT == MVT::nxv16i8) {
5206 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5207 AArch64::MOVA_2ZMXI_V_B);
5209 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5210 VT == MVT::nxv8bf16) {
5211 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5212 AArch64::MOVA_2ZMXI_V_H);
5214 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5215 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5216 AArch64::MOVA_2ZMXI_V_S);
5218 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5219 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5220 AArch64::MOVA_2ZMXI_V_D);
5225 case Intrinsic::aarch64_sme_read_hor_vg4: {
5226 if (VT == MVT::nxv16i8) {
5227 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5228 AArch64::MOVA_4ZMXI_H_B);
5230 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5231 VT == MVT::nxv8bf16) {
5232 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5233 AArch64::MOVA_4ZMXI_H_H);
5235 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5236 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5237 AArch64::MOVA_4ZMXI_H_S);
5239 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5240 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5241 AArch64::MOVA_4ZMXI_H_D);
5246 case Intrinsic::aarch64_sme_read_ver_vg4: {
5247 if (VT == MVT::nxv16i8) {
5248 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5249 AArch64::MOVA_4ZMXI_V_B);
5251 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5252 VT == MVT::nxv8bf16) {
5253 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5254 AArch64::MOVA_4ZMXI_V_H);
5256 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5257 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5258 AArch64::MOVA_4ZMXI_V_S);
5260 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5261 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5262 AArch64::MOVA_4ZMXI_V_D);
5267 case Intrinsic::aarch64_sme_read_vg1x2: {
5268 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5269 AArch64::MOVA_VG2_2ZMXI);
5272 case Intrinsic::aarch64_sme_read_vg1x4: {
5273 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5274 AArch64::MOVA_VG4_4ZMXI);
5277 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5278 if (VT == MVT::nxv16i8) {
5279 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5281 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5282 VT == MVT::nxv8bf16) {
5283 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5285 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5286 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5288 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5289 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5294 case Intrinsic::aarch64_sme_readz_vert_x2: {
5295 if (VT == MVT::nxv16i8) {
5296 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
5298 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5299 VT == MVT::nxv8bf16) {
5300 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
5302 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5303 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
5305 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5306 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
5311 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5312 if (VT == MVT::nxv16i8) {
5313 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
5315 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5316 VT == MVT::nxv8bf16) {
5317 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
5319 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5320 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
5322 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5323 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
5328 case Intrinsic::aarch64_sme_readz_vert_x4: {
5329 if (VT == MVT::nxv16i8) {
5330 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
5332 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5333 VT == MVT::nxv8bf16) {
5334 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
5336 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5337 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
5339 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5340 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
5345 case Intrinsic::swift_async_context_addr: {
5348 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5350 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5351 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5352 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5356 CurDAG->RemoveDeadNode(
Node);
5358 auto &MF = CurDAG->getMachineFunction();
5359 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5363 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5364 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5365 Node->getValueType(0),
5366 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5367 AArch64::LUTI2_4ZTZI_S}))
5369 SelectMultiVectorLuti(
Node, 4, Opc, 3);
5372 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5373 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5374 Node->getValueType(0),
5375 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5377 SelectMultiVectorLuti(
Node, 4, Opc, 1);
5380 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5381 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5382 Node->getValueType(0),
5383 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5384 AArch64::LUTI2_2ZTZI_S}))
5386 SelectMultiVectorLuti(
Node, 2, Opc, 7);
5389 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5390 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5391 Node->getValueType(0),
5392 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5393 AArch64::LUTI4_2ZTZI_S}))
5395 SelectMultiVectorLuti(
Node, 2, Opc, 3);
5401 unsigned IntNo =
Node->getConstantOperandVal(0);
5405 case Intrinsic::aarch64_tagp:
5408 case Intrinsic::aarch64_neon_tbl2:
5409 SelectTable(
Node, 2,
5410 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5413 case Intrinsic::aarch64_neon_tbl3:
5414 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5415 : AArch64::TBLv16i8Three,
5418 case Intrinsic::aarch64_neon_tbl4:
5419 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5420 : AArch64::TBLv16i8Four,
5423 case Intrinsic::aarch64_neon_tbx2:
5424 SelectTable(
Node, 2,
5425 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5428 case Intrinsic::aarch64_neon_tbx3:
5429 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5430 : AArch64::TBXv16i8Three,
5433 case Intrinsic::aarch64_neon_tbx4:
5434 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5435 : AArch64::TBXv16i8Four,
5438 case Intrinsic::aarch64_sve_srshl_single_x2:
5439 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5440 Node->getValueType(0),
5441 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5442 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5443 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5445 case Intrinsic::aarch64_sve_srshl_single_x4:
5446 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5447 Node->getValueType(0),
5448 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5449 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5450 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5452 case Intrinsic::aarch64_sve_urshl_single_x2:
5453 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5454 Node->getValueType(0),
5455 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5456 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5457 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5459 case Intrinsic::aarch64_sve_urshl_single_x4:
5460 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5461 Node->getValueType(0),
5462 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5463 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5464 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5466 case Intrinsic::aarch64_sve_srshl_x2:
5467 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5468 Node->getValueType(0),
5469 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5470 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5471 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5473 case Intrinsic::aarch64_sve_srshl_x4:
5474 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5475 Node->getValueType(0),
5476 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5477 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5478 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5480 case Intrinsic::aarch64_sve_urshl_x2:
5481 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5482 Node->getValueType(0),
5483 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5484 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5485 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5487 case Intrinsic::aarch64_sve_urshl_x4:
5488 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5489 Node->getValueType(0),
5490 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5491 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5492 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5494 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5495 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5496 Node->getValueType(0),
5497 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5498 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5499 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5501 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5502 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5503 Node->getValueType(0),
5504 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5505 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5506 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5508 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5509 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5510 Node->getValueType(0),
5511 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5512 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5513 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5515 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5516 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5517 Node->getValueType(0),
5518 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5519 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5520 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5522 case Intrinsic::aarch64_sve_whilege_x2:
5523 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5524 Node->getValueType(0),
5525 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5526 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5527 SelectWhilePair(
Node,
Op);
5529 case Intrinsic::aarch64_sve_whilegt_x2:
5530 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5531 Node->getValueType(0),
5532 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5533 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5534 SelectWhilePair(
Node,
Op);
5536 case Intrinsic::aarch64_sve_whilehi_x2:
5537 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5538 Node->getValueType(0),
5539 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5540 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5541 SelectWhilePair(
Node,
Op);
5543 case Intrinsic::aarch64_sve_whilehs_x2:
5544 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5545 Node->getValueType(0),
5546 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5547 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5548 SelectWhilePair(
Node,
Op);
5550 case Intrinsic::aarch64_sve_whilele_x2:
5551 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5552 Node->getValueType(0),
5553 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5554 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5555 SelectWhilePair(
Node,
Op);
5557 case Intrinsic::aarch64_sve_whilelo_x2:
5558 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5559 Node->getValueType(0),
5560 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5561 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5562 SelectWhilePair(
Node,
Op);
5564 case Intrinsic::aarch64_sve_whilels_x2:
5565 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5566 Node->getValueType(0),
5567 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5568 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5569 SelectWhilePair(
Node,
Op);
5571 case Intrinsic::aarch64_sve_whilelt_x2:
5572 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5573 Node->getValueType(0),
5574 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5575 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5576 SelectWhilePair(
Node,
Op);
5578 case Intrinsic::aarch64_sve_smax_single_x2:
5579 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5580 Node->getValueType(0),
5581 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5582 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5583 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5585 case Intrinsic::aarch64_sve_umax_single_x2:
5586 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5587 Node->getValueType(0),
5588 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5589 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5590 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5592 case Intrinsic::aarch64_sve_fmax_single_x2:
5593 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5594 Node->getValueType(0),
5595 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
5596 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
5597 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5599 case Intrinsic::aarch64_sve_smax_single_x4:
5600 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5601 Node->getValueType(0),
5602 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5603 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5604 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5606 case Intrinsic::aarch64_sve_umax_single_x4:
5607 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5608 Node->getValueType(0),
5609 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5610 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5611 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5613 case Intrinsic::aarch64_sve_fmax_single_x4:
5614 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5615 Node->getValueType(0),
5616 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
5617 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
5618 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5620 case Intrinsic::aarch64_sve_smin_single_x2:
5621 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5622 Node->getValueType(0),
5623 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5624 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5625 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5627 case Intrinsic::aarch64_sve_umin_single_x2:
5628 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5629 Node->getValueType(0),
5630 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5631 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5632 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5634 case Intrinsic::aarch64_sve_fmin_single_x2:
5635 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5636 Node->getValueType(0),
5637 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
5638 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
5639 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5641 case Intrinsic::aarch64_sve_smin_single_x4:
5642 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5643 Node->getValueType(0),
5644 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5645 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5646 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5648 case Intrinsic::aarch64_sve_umin_single_x4:
5649 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5650 Node->getValueType(0),
5651 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5652 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5653 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5655 case Intrinsic::aarch64_sve_fmin_single_x4:
5656 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5657 Node->getValueType(0),
5658 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
5659 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
5660 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5662 case Intrinsic::aarch64_sve_smax_x2:
5663 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5664 Node->getValueType(0),
5665 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5666 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5667 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5669 case Intrinsic::aarch64_sve_umax_x2:
5670 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5671 Node->getValueType(0),
5672 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5673 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5674 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5676 case Intrinsic::aarch64_sve_fmax_x2:
5677 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5678 Node->getValueType(0),
5679 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
5680 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
5681 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5683 case Intrinsic::aarch64_sve_smax_x4:
5684 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5685 Node->getValueType(0),
5686 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5687 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5688 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5690 case Intrinsic::aarch64_sve_umax_x4:
5691 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5692 Node->getValueType(0),
5693 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5694 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5695 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5697 case Intrinsic::aarch64_sve_fmax_x4:
5698 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5699 Node->getValueType(0),
5700 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
5701 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
5702 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5704 case Intrinsic::aarch64_sve_smin_x2:
5705 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5706 Node->getValueType(0),
5707 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5708 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5709 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5711 case Intrinsic::aarch64_sve_umin_x2:
5712 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5713 Node->getValueType(0),
5714 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5715 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5716 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5718 case Intrinsic::aarch64_sve_fmin_x2:
5719 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5720 Node->getValueType(0),
5721 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
5722 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
5723 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5725 case Intrinsic::aarch64_sve_smin_x4:
5726 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5727 Node->getValueType(0),
5728 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5729 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5730 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5732 case Intrinsic::aarch64_sve_umin_x4:
5733 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5734 Node->getValueType(0),
5735 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
5736 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
5737 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5739 case Intrinsic::aarch64_sve_fmin_x4:
5740 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5741 Node->getValueType(0),
5742 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
5743 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
5744 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5746 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
5747 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5748 Node->getValueType(0),
5749 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
5750 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
5751 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5753 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
5754 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5755 Node->getValueType(0),
5756 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
5757 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
5758 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5760 case Intrinsic::aarch64_sve_fminnm_single_x2:
5761 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5762 Node->getValueType(0),
5763 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
5764 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
5765 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5767 case Intrinsic::aarch64_sve_fminnm_single_x4:
5768 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5769 Node->getValueType(0),
5770 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
5771 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
5772 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5774 case Intrinsic::aarch64_sve_fmaxnm_x2:
5775 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5776 Node->getValueType(0),
5777 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
5778 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
5779 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5781 case Intrinsic::aarch64_sve_fmaxnm_x4:
5782 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5783 Node->getValueType(0),
5784 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
5785 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
5786 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5788 case Intrinsic::aarch64_sve_fminnm_x2:
5789 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5790 Node->getValueType(0),
5791 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
5792 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
5793 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5795 case Intrinsic::aarch64_sve_fminnm_x4:
5796 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5797 Node->getValueType(0),
5798 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
5799 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
5800 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5802 case Intrinsic::aarch64_sve_fcvtzs_x2:
5803 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
5805 case Intrinsic::aarch64_sve_scvtf_x2:
5806 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
5808 case Intrinsic::aarch64_sve_fcvtzu_x2:
5809 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
5811 case Intrinsic::aarch64_sve_ucvtf_x2:
5812 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
5814 case Intrinsic::aarch64_sve_fcvtzs_x4:
5815 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
5817 case Intrinsic::aarch64_sve_scvtf_x4:
5818 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
5820 case Intrinsic::aarch64_sve_fcvtzu_x4:
5821 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
5823 case Intrinsic::aarch64_sve_ucvtf_x4:
5824 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
5826 case Intrinsic::aarch64_sve_fcvt_widen_x2:
5827 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVT_2ZZ_H_S);
5829 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
5830 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVTL_2ZZ_H_S);
5832 case Intrinsic::aarch64_sve_sclamp_single_x2:
5833 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5834 Node->getValueType(0),
5835 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
5836 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
5837 SelectClamp(
Node, 2,
Op);
5839 case Intrinsic::aarch64_sve_uclamp_single_x2:
5840 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5841 Node->getValueType(0),
5842 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
5843 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
5844 SelectClamp(
Node, 2,
Op);
5846 case Intrinsic::aarch64_sve_fclamp_single_x2:
5847 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5848 Node->getValueType(0),
5849 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
5850 AArch64::FCLAMP_VG2_2Z2Z_D}))
5851 SelectClamp(
Node, 2,
Op);
5853 case Intrinsic::aarch64_sve_bfclamp_single_x2:
5854 SelectClamp(
Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
5856 case Intrinsic::aarch64_sve_sclamp_single_x4:
5857 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5858 Node->getValueType(0),
5859 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
5860 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
5861 SelectClamp(
Node, 4,
Op);
5863 case Intrinsic::aarch64_sve_uclamp_single_x4:
5864 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5865 Node->getValueType(0),
5866 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
5867 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
5868 SelectClamp(
Node, 4,
Op);
5870 case Intrinsic::aarch64_sve_fclamp_single_x4:
5871 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5872 Node->getValueType(0),
5873 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
5874 AArch64::FCLAMP_VG4_4Z4Z_D}))
5875 SelectClamp(
Node, 4,
Op);
5877 case Intrinsic::aarch64_sve_bfclamp_single_x4:
5878 SelectClamp(
Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
5880 case Intrinsic::aarch64_sve_add_single_x2:
5881 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5882 Node->getValueType(0),
5883 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
5884 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
5885 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5887 case Intrinsic::aarch64_sve_add_single_x4:
5888 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5889 Node->getValueType(0),
5890 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
5891 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
5892 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5894 case Intrinsic::aarch64_sve_zip_x2:
5895 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5896 Node->getValueType(0),
5897 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
5898 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
5899 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5901 case Intrinsic::aarch64_sve_zipq_x2:
5902 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5903 AArch64::ZIP_VG2_2ZZZ_Q);
5905 case Intrinsic::aarch64_sve_zip_x4:
5906 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5907 Node->getValueType(0),
5908 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
5909 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
5910 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5912 case Intrinsic::aarch64_sve_zipq_x4:
5913 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5914 AArch64::ZIP_VG4_4Z4Z_Q);
5916 case Intrinsic::aarch64_sve_uzp_x2:
5917 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5918 Node->getValueType(0),
5919 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
5920 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
5921 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5923 case Intrinsic::aarch64_sve_uzpq_x2:
5924 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5925 AArch64::UZP_VG2_2ZZZ_Q);
5927 case Intrinsic::aarch64_sve_uzp_x4:
5928 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5929 Node->getValueType(0),
5930 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
5931 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
5932 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5934 case Intrinsic::aarch64_sve_uzpq_x4:
5935 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5936 AArch64::UZP_VG4_4Z4Z_Q);
5938 case Intrinsic::aarch64_sve_sel_x2:
5939 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5940 Node->getValueType(0),
5941 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
5942 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
5943 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
5945 case Intrinsic::aarch64_sve_sel_x4:
5946 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5947 Node->getValueType(0),
5948 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
5949 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
5950 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
5952 case Intrinsic::aarch64_sve_frinta_x2:
5953 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
5955 case Intrinsic::aarch64_sve_frinta_x4:
5956 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
5958 case Intrinsic::aarch64_sve_frintm_x2:
5959 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
5961 case Intrinsic::aarch64_sve_frintm_x4:
5962 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
5964 case Intrinsic::aarch64_sve_frintn_x2:
5965 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
5967 case Intrinsic::aarch64_sve_frintn_x4:
5968 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
5970 case Intrinsic::aarch64_sve_frintp_x2:
5971 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
5973 case Intrinsic::aarch64_sve_frintp_x4:
5974 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
5976 case Intrinsic::aarch64_sve_sunpk_x2:
5977 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5978 Node->getValueType(0),
5979 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
5980 AArch64::SUNPK_VG2_2ZZ_D}))
5981 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5983 case Intrinsic::aarch64_sve_uunpk_x2:
5984 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5985 Node->getValueType(0),
5986 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
5987 AArch64::UUNPK_VG2_2ZZ_D}))
5988 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5990 case Intrinsic::aarch64_sve_sunpk_x4:
5991 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5992 Node->getValueType(0),
5993 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
5994 AArch64::SUNPK_VG4_4Z2Z_D}))
5995 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5997 case Intrinsic::aarch64_sve_uunpk_x4:
5998 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5999 Node->getValueType(0),
6000 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6001 AArch64::UUNPK_VG4_4Z2Z_D}))
6002 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6004 case Intrinsic::aarch64_sve_pext_x2: {
6005 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6006 Node->getValueType(0),
6007 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6008 AArch64::PEXT_2PCI_D}))
6009 SelectPExtPair(
Node,
Op);
6016 unsigned IntNo =
Node->getConstantOperandVal(1);
6017 if (
Node->getNumOperands() >= 3)
6018 VT =
Node->getOperand(2)->getValueType(0);
6022 case Intrinsic::aarch64_neon_st1x2: {
6023 if (VT == MVT::v8i8) {
6024 SelectStore(
Node, 2, AArch64::ST1Twov8b);
6026 }
else if (VT == MVT::v16i8) {
6027 SelectStore(
Node, 2, AArch64::ST1Twov16b);
6029 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6030 VT == MVT::v4bf16) {
6031 SelectStore(
Node, 2, AArch64::ST1Twov4h);
6033 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6034 VT == MVT::v8bf16) {
6035 SelectStore(
Node, 2, AArch64::ST1Twov8h);
6037 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6038 SelectStore(
Node, 2, AArch64::ST1Twov2s);
6040 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6041 SelectStore(
Node, 2, AArch64::ST1Twov4s);
6043 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6044 SelectStore(
Node, 2, AArch64::ST1Twov2d);
6046 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6047 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6052 case Intrinsic::aarch64_neon_st1x3: {
6053 if (VT == MVT::v8i8) {
6054 SelectStore(
Node, 3, AArch64::ST1Threev8b);
6056 }
else if (VT == MVT::v16i8) {
6057 SelectStore(
Node, 3, AArch64::ST1Threev16b);
6059 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6060 VT == MVT::v4bf16) {
6061 SelectStore(
Node, 3, AArch64::ST1Threev4h);
6063 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6064 VT == MVT::v8bf16) {
6065 SelectStore(
Node, 3, AArch64::ST1Threev8h);
6067 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6068 SelectStore(
Node, 3, AArch64::ST1Threev2s);
6070 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6071 SelectStore(
Node, 3, AArch64::ST1Threev4s);
6073 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6074 SelectStore(
Node, 3, AArch64::ST1Threev2d);
6076 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6077 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6082 case Intrinsic::aarch64_neon_st1x4: {
6083 if (VT == MVT::v8i8) {
6084 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
6086 }
else if (VT == MVT::v16i8) {
6087 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
6089 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6090 VT == MVT::v4bf16) {
6091 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
6093 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6094 VT == MVT::v8bf16) {
6095 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
6097 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6098 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
6100 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6101 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
6103 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6104 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
6106 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6107 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6112 case Intrinsic::aarch64_neon_st2: {
6113 if (VT == MVT::v8i8) {
6114 SelectStore(
Node, 2, AArch64::ST2Twov8b);
6116 }
else if (VT == MVT::v16i8) {
6117 SelectStore(
Node, 2, AArch64::ST2Twov16b);
6119 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6120 VT == MVT::v4bf16) {
6121 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6123 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6124 VT == MVT::v8bf16) {
6125 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6127 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6128 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6130 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6131 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6133 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6134 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6136 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6137 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6142 case Intrinsic::aarch64_neon_st3: {
6143 if (VT == MVT::v8i8) {
6144 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6146 }
else if (VT == MVT::v16i8) {
6147 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6149 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6150 VT == MVT::v4bf16) {
6151 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6153 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6154 VT == MVT::v8bf16) {
6155 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6157 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6158 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6160 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6161 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6163 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6164 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6166 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6167 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6172 case Intrinsic::aarch64_neon_st4: {
6173 if (VT == MVT::v8i8) {
6174 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6176 }
else if (VT == MVT::v16i8) {
6177 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6179 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6180 VT == MVT::v4bf16) {
6181 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6183 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6184 VT == MVT::v8bf16) {
6185 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6187 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6188 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6190 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6191 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6193 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6194 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6196 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6197 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6202 case Intrinsic::aarch64_neon_st2lane: {
6203 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6204 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6206 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6207 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6208 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6210 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6212 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6214 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6216 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6221 case Intrinsic::aarch64_neon_st3lane: {
6222 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6223 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6225 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6226 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6227 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6229 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6231 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6233 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6235 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6240 case Intrinsic::aarch64_neon_st4lane: {
6241 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6242 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6244 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6245 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6246 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6248 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6250 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6252 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6254 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6259 case Intrinsic::aarch64_sve_st2q: {
6260 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6263 case Intrinsic::aarch64_sve_st3q: {
6264 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6267 case Intrinsic::aarch64_sve_st4q: {
6268 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6271 case Intrinsic::aarch64_sve_st2: {
6272 if (VT == MVT::nxv16i8) {
6273 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6275 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6276 VT == MVT::nxv8bf16) {
6277 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6279 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6280 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6282 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6283 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6288 case Intrinsic::aarch64_sve_st3: {
6289 if (VT == MVT::nxv16i8) {
6290 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6292 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6293 VT == MVT::nxv8bf16) {
6294 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6296 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6297 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6299 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6300 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6305 case Intrinsic::aarch64_sve_st4: {
6306 if (VT == MVT::nxv16i8) {
6307 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6309 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6310 VT == MVT::nxv8bf16) {
6311 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6313 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6314 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6316 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6317 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6326 if (VT == MVT::v8i8) {
6327 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6329 }
else if (VT == MVT::v16i8) {
6330 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6332 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6333 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6335 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6336 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6338 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6339 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6341 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6342 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6344 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6345 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6347 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6348 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6354 if (VT == MVT::v8i8) {
6355 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6357 }
else if (VT == MVT::v16i8) {
6358 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6360 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6361 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6363 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6364 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6366 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6367 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6369 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6370 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6372 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6373 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6375 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6376 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6382 if (VT == MVT::v8i8) {
6383 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6385 }
else if (VT == MVT::v16i8) {
6386 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6388 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6389 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6391 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6392 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6394 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6395 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6397 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6398 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6400 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6401 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6403 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6404 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6410 if (VT == MVT::v8i8) {
6411 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6413 }
else if (VT == MVT::v16i8) {
6414 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6416 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6417 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6419 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6420 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6422 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6423 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6425 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6426 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6428 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6429 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6431 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6432 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6438 if (VT == MVT::v8i8) {
6439 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6441 }
else if (VT == MVT::v16i8) {
6442 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6444 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6445 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6447 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6448 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6450 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6451 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6453 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6454 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6456 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6457 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6459 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6460 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6466 if (VT == MVT::v8i8) {
6467 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6469 }
else if (VT == MVT::v16i8) {
6470 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6472 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6473 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6475 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6476 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6478 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6479 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6481 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6482 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6484 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6485 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6487 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6488 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6494 if (VT == MVT::v8i8) {
6495 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6497 }
else if (VT == MVT::v16i8) {
6498 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6500 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6501 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6503 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6504 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6506 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6507 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6509 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6510 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6512 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6513 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6515 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6516 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6522 if (VT == MVT::v8i8) {
6523 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6525 }
else if (VT == MVT::v16i8) {
6526 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6528 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6529 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6531 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6532 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6534 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6535 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6537 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6538 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6540 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6541 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6543 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6544 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6550 if (VT == MVT::v8i8) {
6551 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6553 }
else if (VT == MVT::v16i8) {
6554 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6556 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6557 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6559 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6560 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6562 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6563 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6565 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6566 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6568 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6569 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6571 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6572 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6578 if (VT == MVT::v8i8) {
6579 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6581 }
else if (VT == MVT::v16i8) {
6582 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6584 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6585 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6587 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6588 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6590 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6591 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6593 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6594 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6596 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6597 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6599 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6600 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6606 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6607 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6609 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6610 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6611 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6613 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6615 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6617 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6619 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6625 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6626 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6628 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6629 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6630 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6632 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6634 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6636 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6638 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6644 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6645 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6647 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6648 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6649 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6651 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6653 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6655 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6657 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6663 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6664 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6666 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6667 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6668 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6670 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6672 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6674 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6676 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6682 VT =
Node->getOperand(1).getValueType();
6683 if (VT == MVT::v8i8) {
6684 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6686 }
else if (VT == MVT::v16i8) {
6687 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6689 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6690 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6692 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6693 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6695 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6696 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6698 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6699 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6701 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6702 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6704 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6705 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6711 VT =
Node->getOperand(1).getValueType();
6712 if (VT == MVT::v8i8) {
6713 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6715 }
else if (VT == MVT::v16i8) {
6716 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6718 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6719 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6721 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6722 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6724 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6725 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6727 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6728 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6730 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6731 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
6733 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6734 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6740 VT =
Node->getOperand(1).getValueType();
6741 if (VT == MVT::v8i8) {
6742 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
6744 }
else if (VT == MVT::v16i8) {
6745 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
6747 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6748 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
6750 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6751 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
6753 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6754 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
6756 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6757 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
6759 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6760 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
6762 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6763 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6769 VT =
Node->getOperand(1).getValueType();
6770 if (VT == MVT::v8i8) {
6771 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
6773 }
else if (VT == MVT::v16i8) {
6774 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
6776 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6777 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
6779 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6780 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
6782 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6783 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
6785 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6786 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
6788 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6789 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6791 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6792 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
6798 VT =
Node->getOperand(1).getValueType();
6799 if (VT == MVT::v8i8) {
6800 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
6802 }
else if (VT == MVT::v16i8) {
6803 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
6805 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6806 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
6808 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
6809 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
6811 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6812 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
6814 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6815 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
6817 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6818 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6820 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6821 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
6827 VT =
Node->getOperand(1).getValueType();
6828 if (VT == MVT::v8i8) {
6829 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
6831 }
else if (VT == MVT::v16i8) {
6832 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
6834 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6835 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
6837 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6838 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
6840 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6841 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
6843 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6844 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
6846 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6847 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6849 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6850 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
6856 VT =
Node->getOperand(1).getValueType();
6857 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6858 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
6860 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6861 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6862 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
6864 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6866 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
6868 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6870 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
6876 VT =
Node->getOperand(1).getValueType();
6877 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6878 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
6880 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6881 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6882 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
6884 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6886 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
6888 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6890 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
6896 VT =
Node->getOperand(1).getValueType();
6897 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6898 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
6900 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6901 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6902 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
6904 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6906 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
6908 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6910 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
6916 if (VT == MVT::nxv16i8) {
6917 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
6919 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6920 VT == MVT::nxv8bf16) {
6921 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
6923 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6924 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
6926 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6927 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
6933 if (VT == MVT::nxv16i8) {
6934 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
6936 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6937 VT == MVT::nxv8bf16) {
6938 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
6940 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6941 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
6943 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6944 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
6950 if (VT == MVT::nxv16i8) {
6951 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
6953 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6954 VT == MVT::nxv8bf16) {
6955 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
6957 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6958 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
6960 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6961 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
6976 return new AArch64DAGToDAGISelLegacy(
TM, OptLevel);
6988 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
6992 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
6993 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7007 if (isa<MemSDNode>(Root))
7008 return cast<MemSDNode>(Root)->getMemoryVT();
7010 if (isa<MemIntrinsicSDNode>(Root))
7011 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
7013 const unsigned Opcode = Root->
getOpcode();
7021 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
7023 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
7043 case Intrinsic::aarch64_sme_ldr:
7044 case Intrinsic::aarch64_sme_str:
7045 return MVT::nxv16i8;
7046 case Intrinsic::aarch64_sve_prf:
7051 case Intrinsic::aarch64_sve_ld2_sret:
7052 case Intrinsic::aarch64_sve_ld2q_sret:
7055 case Intrinsic::aarch64_sve_st2q:
7058 case Intrinsic::aarch64_sve_ld3_sret:
7059 case Intrinsic::aarch64_sve_ld3q_sret:
7062 case Intrinsic::aarch64_sve_st3q:
7065 case Intrinsic::aarch64_sve_ld4_sret:
7066 case Intrinsic::aarch64_sve_ld4q_sret:
7069 case Intrinsic::aarch64_sve_st4q:
7072 case Intrinsic::aarch64_sve_ld1udq:
7073 case Intrinsic::aarch64_sve_st1dq:
7074 return EVT(MVT::nxv1i64);
7075 case Intrinsic::aarch64_sve_ld1uwq:
7076 case Intrinsic::aarch64_sve_st1wq:
7077 return EVT(MVT::nxv1i32);
7084template <
int64_t Min,
int64_t Max>
7085bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
7093 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
7098 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
7117 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
7119 if ((MulImm % MemWidthBytes) != 0)
7122 int64_t
Offset = MulImm / MemWidthBytes;
7123 if (Offset < Min || Offset > Max)
7126 Base =
N.getOperand(0);
7128 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7135 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7141bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7159 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7160 int64_t ImmOff =
C->getSExtValue();
7161 unsigned Size = 1 << Scale;
7170 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7172 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7182 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7183 if (
C->getZExtValue() == Scale) {
7192bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7199bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7200 EVT VT =
N.getValueType();
7204bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7209 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7210 int64_t ImmOff =
C->getSExtValue();
7211 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7212 Base =
N.getOperand(0);
7213 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7220 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
unsigned const TargetRegisterInfo * TRI
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
static constexpr unsigned MaxRecursionDepth
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Not(const Pred &P) -> Not< Pred >
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.