25#include "llvm/IR/IntrinsicsAArch64.h"
36#define DEBUG_TYPE "aarch64-isel"
37#define PASS_NAME "AArch64 Instruction Selection"
40#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
41#pragma inline_depth(0)
57 AArch64DAGToDAGISel() =
delete;
69 void PreprocessISelDAG()
override;
73 bool SelectInlineAsmMemoryOperand(
const SDValue &
Op,
75 std::vector<SDValue> &OutOps)
override;
77 template <
signed Low,
signed High,
signed Scale>
80 template <
signed Low,
signed High>
88 return SelectShiftedRegister(
N,
false,
Reg, Shift);
91 return SelectShiftedRegister(
N,
true,
Reg, Shift);
93 template <
unsigned ShiftW
idth>
97 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
100 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
103 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
106 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
109 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
112 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
115 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
118 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
121 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
124 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
127 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
130 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
133 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
136 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
139 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
142 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
145 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
147 template <
unsigned Size,
unsigned Max>
151 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
154 int64_t
C = CI->getSExtValue();
162 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
169 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
175 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
180 N =
N->getOperand(0);
184 EVT VT =
N->getValueType(0);
185 EVT LVT =
N->getOperand(0).getValueType();
186 unsigned Index =
N->getConstantOperandVal(1);
190 Res =
N->getOperand(0);
195 if (
N.getOpcode() != AArch64ISD::VLSHR)
198 EVT VT =
Op.getValueType();
199 unsigned ShtAmt =
N->getConstantOperandVal(1);
204 if (
Op.getOperand(1).getOpcode() == AArch64ISD::MOVIshift)
206 Op.getOperand(1).getConstantOperandVal(0)
207 <<
Op.getOperand(1).getConstantOperandVal(1));
208 else if (
Op.getOperand(1).getOpcode() == AArch64ISD::DUP &&
211 Op.getOperand(1).getConstantOperandVal(0));
215 if (
Imm != 1ULL << (ShtAmt - 1))
218 Res1 =
Op.getOperand(0);
219 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
223 bool SelectDupZeroOrUndef(
SDValue N) {
224 switch(
N->getOpcode()) {
228 case AArch64ISD::DUP:
230 auto Opnd0 =
N->getOperand(0);
244 bool SelectAny(
SDValue) {
return true; }
247 switch(
N->getOpcode()) {
248 case AArch64ISD::DUP:
250 auto Opnd0 =
N->getOperand(0);
262 template <MVT::SimpleValueType VT,
bool Negate>
264 return SelectSVEAddSubImm(
N, VT,
Imm, Shift, Negate);
267 template <MVT::SimpleValueType VT,
bool Negate>
269 return SelectSVEAddSubSSatImm(
N, VT,
Imm, Shift, Negate);
272 template <MVT::SimpleValueType VT>
274 return SelectSVECpyDupImm(
N, VT,
Imm, Shift);
277 template <MVT::SimpleValueType VT,
bool Invert = false>
279 return SelectSVELogicalImm(
N, VT,
Imm, Invert);
282 template <MVT::SimpleValueType VT>
284 return SelectSVEArithImm(
N, VT,
Imm);
287 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
289 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation,
Imm);
296 EVT EltVT =
N->getValueType(0).getVectorElementType();
297 return SelectSVEShiftImm(
N->getOperand(0), 1,
303 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
310 MulImm = 1LL << MulImm;
312 if ((MulImm % std::abs(Scale)) != 0)
316 if ((MulImm >= Min) && (MulImm <= Max)) {
317 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
324 template <
signed Max,
signed Scale>
331 if (MulImm >= 0 && MulImm <= Max) {
333 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
340 template <
unsigned BaseReg,
unsigned Max>
348 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
371 const unsigned SubRegs[]);
373 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
375 bool tryIndexedLoad(
SDNode *
N);
377 void SelectPtrauthAuth(
SDNode *
N);
378 void SelectPtrauthResign(
SDNode *
N);
379 void SelectPtrauthResignWithPC(
SDNode *
N);
381 bool trySelectStackSlotTagP(
SDNode *
N);
384 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
386 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
388 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
389 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
390 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
391 unsigned Opc_rr,
unsigned Opc_ri,
392 bool IsIntr =
false);
393 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
394 unsigned Scale,
unsigned Opc_ri,
396 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
397 bool IsZmMulti,
unsigned Opcode,
398 bool HasPred =
false);
400 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
401 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
402 void SelectCVTIntrinsicFP8(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
403 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
404 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
405 bool IsTupleInput,
unsigned Opc);
406 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
408 template <
unsigned MaxIdx,
unsigned Scale>
409 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
411 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
412 unsigned Op,
unsigned MaxIdx,
unsigned Scale,
413 unsigned BaseReg = 0);
415 template <
int64_t Min,
int64_t Max>
419 template <
unsigned Scale>
421 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
424 void SelectMultiVectorLutiLane(
SDNode *
Node,
unsigned NumOutVecs,
426 void SelectMultiVectorLuti6LaneX4(
SDNode *
Node,
unsigned NumIndexVecs);
428 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
431 template <
unsigned MaxIdx,
unsigned Scale>
436 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
437 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
438 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
439 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
440 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
441 unsigned Opc_rr,
unsigned Opc_ri);
442 std::tuple<unsigned, SDValue, SDValue>
443 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
447 bool tryBitfieldExtractOp(
SDNode *
N);
448 bool tryBitfieldInsertOp(
SDNode *
N);
449 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
451 bool tryReadRegister(
SDNode *
N);
452 bool tryWriteRegister(
SDNode *
N);
454 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
455 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
459 bool tryFoldCselToFMaxMin(
SDNode *
N);
462#include "AArch64GenDAGISel.inc"
470 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
472 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
485 bool isWorthNegatingImm(
SDValue V)
const;
486 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
487 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
488 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
491 template<
unsigned RegW
idth>
493 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
495 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
497 template <
unsigned RegW
idth>
499 return SelectCVTFixedPointVec(
N, FixedPos, RegWidth);
501 bool SelectCVTFixedPointVec(
SDValue N,
SDValue &FixedPos,
unsigned Width);
503 template<
unsigned RegW
idth>
505 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
511 template <
unsigned FloatW
idth>
513 return SelectCVTFixedPosRecipOperandVec(
N, FixedPos, FloatWidth);
519 bool SelectCMP_SWAP(
SDNode *
N);
551 bool SelectAllActivePredicate(
SDValue N);
556 template <
bool MatchCBB>
566 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
570char AArch64DAGToDAGISelLegacy::ID = 0;
576 std::make_unique<AArch64DAGToDAGISel>(TM, TM.getOptLevel())) {}
582 auto getFloatVT = [&](
EVT VT) {
584 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) &&
"Unexpected VT");
585 return VT.changeElementType(*(DAG.
getContext()),
586 ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
591 for (
unsigned I = 0,
E =
N.getNumOperands();
I <
E; ++
I) {
592 auto bitcasted = DAG.
getBitcast(getFloatVT(
N.getOperand(
I).getValueType()),
596 EVT OrigVT =
N.getValueType(0);
605 Imm =
C->getZExtValue();
622 return N->getOpcode() ==
Opc &&
633 return Imm == ImmExpected;
638 assert(RegWidth == 32 || RegWidth == 64);
640 return APInt(RegWidth,
648 assert(
N.getValueType().isInteger() &&
"Only integers are supported");
649 if (
N->getOpcode() == AArch64ISD::NVCAST ||
651 N =
N->getOperand(0);
652 unsigned SplatWidth =
N.getScalarValueSizeInBits();
653 if (
N.getOpcode() == AArch64ISD::FMOV)
655 if (
N->getOpcode() == AArch64ISD::MOVI)
656 return APInt(SplatWidth,
N.getConstantOperandVal(0));
657 if (
N->getOpcode() == AArch64ISD::MOVIshift)
658 return APInt(SplatWidth,
N.getConstantOperandVal(0)
659 <<
N.getConstantOperandVal(1));
660 if (
N->getOpcode() == AArch64ISD::MVNIshift)
661 return ~APInt(SplatWidth,
N.getConstantOperandVal(0)
662 <<
N.getConstantOperandVal(1));
663 if (
N->getOpcode() == AArch64ISD::MOVIedit)
665 N.getConstantOperandVal(0)));
666 if (
N->getOpcode() == AArch64ISD::DUP)
668 return Const->getAPIntValue().trunc(SplatWidth);
671 return SplatVal.
trunc(SplatWidth);
679static std::optional<APInt>
681 unsigned SplatWidth =
N.getScalarValueSizeInBits();
683 if (SplatVal->getBitWidth() <= SplatWidth)
685 if (SplatVal->isSplat(SplatWidth))
686 return SplatVal->trunc(SplatWidth);
691bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(
SDValue N,
698 ImmVal->getZExtValue(), Encoding))
701 Imm = CurDAG->getTargetConstant(Encoding, SDLoc(
N), MVT::i64);
708 return SelectSVEAddSubImm(SDLoc(
N), *ImmVal,
709 N.getValueType().getScalarType().getSimpleVT(),
715bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(
SDValue N,
718 return SelectSVESignedArithImm(SDLoc(
N), *ImmVal,
Imm);
727 int64_t ImmVal = ImmAPIntVal->getSExtValue();
728 if (ImmVal < -128 || ImmVal > 127)
731 Imm = CurDAG->getSignedTargetConstant(ImmVal, SDLoc(
N), MVT::i32);
740 uint64_t ImmVal = ImmAPIntVal->getZExtValue();
744 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(
N), MVT::i32);
748bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
750 std::vector<SDValue> &OutOps) {
751 switch(ConstraintID) {
754 case InlineAsm::ConstraintCode::m:
755 case InlineAsm::ConstraintCode::o:
756 case InlineAsm::ConstraintCode::Q:
763 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
765 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
766 dl,
Op.getValueType(),
768 OutOps.push_back(NewOp);
780 switch (User->getOpcode()) {
785 return User->getOperand(1).getNode() == N.getNode();
786 case ISD::ANY_EXTEND:
787 case ISD::ZERO_EXTEND:
788 case ISD::SIGN_EXTEND:
791 return allUsesAreShiftAmounts(SDValue(User, 0));
798template <
unsigned ShiftW
idth>
808 (
N.getValueType() == MVT::i32 ||
N.getValueType() == MVT::i64)) {
820 (
N.getValueType() == MVT::i32 ||
N.getValueType() == MVT::i64)) {
823 (
Imm % ShiftWidth == 0)) {
832 (
N.getValueType() == MVT::i32 ||
N.getValueType() == MVT::i64)) {
835 (
Imm % ShiftWidth == 0)) {
837 EVT VT =
N.getValueType();
838 bool Is32 = VT == MVT::i32;
839 unsigned NegOpc = Is32 ? AArch64::SUBWrr : AArch64::SUBXrr;
840 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
842 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, VT);
844 CurDAG->getMachineNode(NegOpc,
DL, VT,
Zero,
N.getOperand(1));
853 (
N.getValueType() == MVT::i32 ||
N.getValueType() == MVT::i64)) {
856 (
Imm % ShiftWidth == ShiftWidth - 1)) {
858 EVT VT =
N.getValueType();
859 bool Is32 = VT == MVT::i32;
860 unsigned NotOpc = Is32 ? AArch64::ORNWrr : AArch64::ORNXrr;
861 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
863 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, VT);
865 CurDAG->getMachineNode(NotOpc,
DL, VT,
Zero,
N.getOperand(1));
887 uint64_t Immed =
N.getNode()->getAsZExtVal();
897 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
898 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
915 uint64_t Immed =
N.getNode()->getAsZExtVal();
923 if (
N.getValueType() == MVT::i32)
924 Immed = ~((uint32_t)Immed) + 1;
926 Immed = ~Immed + 1ULL;
927 if (Immed & 0xFFFFFFFFFF000000ULL)
930 Immed &= 0xFFFFFFULL;
931 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(
N), MVT::i32), Val,
938 switch (
N.getOpcode()) {
964 unsigned ShiftVal = CSD->getZExtValue();
982bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
985 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
1014 EVT VT =
N.getValueType();
1015 if (VT != MVT::i32 && VT != MVT::i64)
1018 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
1024 unsigned LHSOpcode =
LHS->getOpcode();
1038 unsigned LowZBits, MaskLen;
1042 unsigned BitWidth =
N.getValueSizeInBits();
1045 unsigned NewShiftOp;
1048 if (LowZBits <= ShiftAmtC)
1051 NewShiftC = LowZBits - ShiftAmtC;
1052 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1058 NewShiftC = LowZBits + ShiftAmtC;
1067 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1069 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
1073 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
1077 MSB = std::min<uint64_t>(NewShiftC + MaskLen,
BitWidth) - 1;
1078 SDValue MSBC = CurDAG->getTargetConstant(MSB,
DL, VT);
1079 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
1083 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
1097 SrcVT =
N.getOperand(0).getValueType();
1099 if (!IsLoadStore && SrcVT == MVT::i8)
1101 else if (!IsLoadStore && SrcVT == MVT::i16)
1103 else if (SrcVT == MVT::i32)
1105 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
1110 EVT SrcVT =
N.getOperand(0).getValueType();
1111 if (!IsLoadStore && SrcVT == MVT::i8)
1113 else if (!IsLoadStore && SrcVT == MVT::i16)
1115 else if (SrcVT == MVT::i32)
1117 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
1142bool AArch64DAGToDAGISel::isWorthNegatingImm(
SDValue V)
const {
1145 EVT VT =
V.getValueType();
1146 assert((VT == MVT::i32 || VT == MVT::i64) &&
"invalid type");
1157 return NewCost.
size() < OrigCost.
size();
1164bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
1167 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
1173 V.getConstantOperandVal(1) <= 4 &&
1205 Opd0 =
N->getOperand(0).getOperand(0);
1215 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
1222 unsigned &Immr,
unsigned &Imms,
1223 bool BiggerPattern) {
1225 "N must be a SHR/SRA operation to call this function");
1227 EVT VT =
N->getValueType(0);
1232 assert((VT == MVT::i32 || VT == MVT::i64) &&
1233 "Type checking must have been done before calling this function");
1243 Opd0 =
N->getOperand(0).getOperand(0);
1244 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
1250 Opd0 =
N->getOperand(0).getOperand(0);
1253 assert(VT == MVT::i64 &&
"the promoted type should be i64");
1254 }
else if (BiggerPattern) {
1258 Opd0 =
N->getOperand(0);
1267 <<
": Found large shift immediate, this should not happen\n"));
1276 "bad amount in shift node!");
1277 int immr = SrlImm - ShlImm;
1282 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
1284 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
1293bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
1295 if (SelectShiftedRegisterFromAnd(
N,
Reg, Shift))
1308 unsigned Opc, Immr, Imms;
1319 unsigned BitSize =
N.getValueSizeInBits();
1320 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
1323 Reg =
N.getOperand(0);
1324 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(
N), MVT::i32);
1325 return isWorthFoldingALU(
N,
true);
1336 if (
N.getValueType() == MVT::i32)
1344template<
signed Low,
signed High,
signed Scale>
1350 if ((MulImm % std::abs(Scale)) == 0) {
1351 int64_t RDVLImm = MulImm / Scale;
1352 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
1353 Imm = CurDAG->getSignedTargetConstant(RDVLImm, SDLoc(
N), MVT::i32);
1362template <
signed Low,
signed High>
1368 if (MulImm >=
Low && MulImm <=
High) {
1369 Imm = CurDAG->getSignedTargetConstant(MulImm, SDLoc(
N), MVT::i32);
1380 unsigned ShiftVal = 0;
1395 Reg =
N.getOperand(0).getOperand(0);
1407 Op =
Op->getOperand(0);
1409 Op.getOperand(0).getValueType().isFixedLengthVector())
1413 Reg =
N.getOperand(0);
1418 unsigned Opc =
N.getOpcode();
1431 Reg.getValueType() == MVT::i32 &&
1442 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(
N),
1444 return isWorthFoldingALU(
N);
1451 unsigned ShiftVal = 0;
1465 Reg =
N.getOperand(0);
1466 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(
N),
1468 return isWorthFoldingALU(
N);
1477 for (
auto *
User :
N->users()) {
1504bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1505 unsigned BW,
unsigned Size,
1509 const DataLayout &
DL = CurDAG->getDataLayout();
1510 const TargetLowering *
TLI = getTargetLowering();
1514 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1520 if (CurDAG->isBaseWithConstantOffset(
N)) {
1523 int64_t RHSC =
RHS->getSExtValue();
1525 int64_t
Range = 0x1LL << (BW - 1);
1527 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1528 RHSC < (
Range << Scale)) {
1529 Base =
N.getOperand(0);
1534 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1543 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1544 Base =
N.getOperand(0);
1549 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1560 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1567bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1570 const DataLayout &
DL = CurDAG->getDataLayout();
1571 const TargetLowering *
TLI = getTargetLowering();
1575 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1580 GlobalAddressSDNode *GAN =
1582 Base =
N.getOperand(0);
1592 if (CurDAG->isBaseWithConstantOffset(
N)) {
1594 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1597 Base =
N.getOperand(0);
1602 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1610 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1618 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1627bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1630 if (!CurDAG->isBaseWithConstantOffset(
N))
1633 int64_t RHSC =
RHS->getSExtValue();
1634 if (RHSC >= -256 && RHSC < 256) {
1635 Base =
N.getOperand(0);
1638 const TargetLowering *
TLI = getTargetLowering();
1639 Base = CurDAG->getTargetFrameIndex(
1642 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(
N), MVT::i64);
1652 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1659bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1679 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1685 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1688 return isWorthFoldingAddr(
N,
Size);
1691bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1709 const SDNode *
Node =
N.getNode();
1710 for (SDNode *UI :
Node->users()) {
1716 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1719 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1722 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1727 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1730 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1735 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1739 if (IsExtendedRegisterWorthFolding &&
1746 if (isWorthFoldingAddr(
LHS,
Size))
1751 if (IsExtendedRegisterWorthFolding &&
1758 if (isWorthFoldingAddr(
RHS,
Size))
1770 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1773 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1775 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1776 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1780bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1793 const SDNode *
Node =
N.getNode();
1794 for (SDNode *UI :
Node->users()) {
1811 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1821 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64,
Ops);
1828 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1831 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1834 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1839 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1842 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1849 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1850 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1856 static const unsigned RegClassIDs[] = {
1857 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1858 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1859 AArch64::dsub2, AArch64::dsub3};
1865 static const unsigned RegClassIDs[] = {
1866 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1867 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1868 AArch64::qsub2, AArch64::qsub3};
1874 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1875 AArch64::ZPR3RegClassID,
1876 AArch64::ZPR4RegClassID};
1877 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1878 AArch64::zsub2, AArch64::zsub3};
1888 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1889 AArch64::ZPR4Mul4RegClassID};
1890 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1891 AArch64::zsub2, AArch64::zsub3};
1896 const unsigned RegClassIDs[],
1897 const unsigned SubRegs[]) {
1900 if (Regs.
size() == 1)
1911 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1914 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1915 Ops.push_back(Regs[i]);
1916 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1920 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped,
Ops);
1924void AArch64DAGToDAGISel::SelectTable(SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1927 EVT VT =
N->getValueType(0);
1929 unsigned ExtOff = isExt;
1932 unsigned Vec0Off = ExtOff + 1;
1938 Ops.push_back(
N->getOperand(1));
1939 Ops.push_back(RegSeq);
1940 Ops.push_back(
N->getOperand(NumVecs + ExtOff + 1));
1941 ReplaceNode(
N, CurDAG->getMachineNode(
Opc, dl, VT,
Ops));
1944static std::tuple<SDValue, SDValue>
1965 if (!ConstDiscN || !
isUInt<16>(ConstDiscN->getZExtValue()))
1970 AddrDisc = DAG->
getRegister(AArch64::XZR, MVT::i64);
1972 return std::make_tuple(
1977void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *
N) {
1982 SDValue AUTDisc =
N->getOperand(3);
1985 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1987 SDValue AUTAddrDisc, AUTConstDisc;
1988 std::tie(AUTConstDisc, AUTAddrDisc) =
1992 std::vector<SDValue>
Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1994 if (
N->getNumOperands() > 4)
1995 Ops.push_back(
N->getOperand(4));
1998 CurDAG->getMachineNode(AArch64::AUTxMxN,
DL, MVT::i64, MVT::i64,
Ops);
1999 ReplaceNode(
N, AUT);
2001 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
2002 AArch64::X16, Val,
SDValue());
2005 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTx16x17,
DL, MVT::i64,
Ops);
2006 ReplaceNode(
N, AUT);
2010void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *
N) {
2020 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
2025 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
2026 PACKey = CurDAG->getTargetConstant(PACKeyC,
DL, MVT::i64);
2028 SDValue AUTAddrDisc, AUTConstDisc;
2029 std::tie(AUTConstDisc, AUTAddrDisc) =
2032 SDValue PACAddrDisc, PACConstDisc;
2033 std::tie(PACConstDisc, PACAddrDisc) =
2036 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
2037 AArch64::X16, Val,
SDValue());
2040 SDValue Addend =
N->getOperand(OffsetBase + 6);
2041 SDValue IncomingChain =
N->getOperand(0);
2042 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
2043 PACKey, PACConstDisc, PACAddrDisc,
2044 Addend, IncomingChain, X16Copy.
getValue(1)};
2046 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(AArch64::AUTRELLOADPAC,
DL,
2047 MVT::i64, MVT::Other,
Ops);
2048 ReplaceNode(
N, AUTRELLOADPAC);
2050 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
2051 PACConstDisc, PACAddrDisc, X16Copy.
getValue(1)};
2053 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC,
DL, MVT::i64,
Ops);
2054 ReplaceNode(
N, AUTPAC);
2058void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *
N) {
2070 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
2071 PACKey = CurDAG->getTargetConstant(PACKeyC,
DL, MVT::i64);
2073 SDValue PACAddrDisc, PACConstDisc;
2074 std::tie(PACConstDisc, PACAddrDisc) =
2077 SDValue X17Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
2078 AArch64::X17, Val,
SDValue());
2079 SDValue X16Copy = CurDAG->getCopyToReg(
2080 CurDAG->getEntryNode(),
DL, AArch64::X16, AUTDisc, X17Copy.
getValue(1));
2081 SDValue X15Copy = CurDAG->getCopyToReg(
2082 CurDAG->getEntryNode(),
DL, AArch64::X15, AUTPC, X16Copy.
getValue(1));
2084 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
2087 CurDAG->getMachineNode(AArch64::AUTPCPAC,
DL, MVT::i64,
Ops);
2088 ReplaceNode(
N, AUTPCPAC);
2091bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *
N) {
2093 if (
LD->isUnindexed())
2095 EVT VT =
LD->getMemoryVT();
2096 EVT DstVT =
N->getValueType(0);
2100 int OffsetVal = (int)
OffsetOp->getZExtValue();
2105 unsigned Opcode = 0;
2108 bool InsertTo64 =
false;
2114 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
2115 else if (VT == MVT::i32) {
2117 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
2119 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
2121 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
2127 }
else if (VT == MVT::i16) {
2129 if (DstVT == MVT::i64)
2130 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
2132 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
2134 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
2135 InsertTo64 = DstVT == MVT::i64;
2140 }
else if (VT == MVT::i8) {
2142 if (DstVT == MVT::i64)
2143 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
2145 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
2147 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
2148 InsertTo64 = DstVT == MVT::i64;
2153 }
else if (VT == MVT::f16) {
2154 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2155 }
else if (VT == MVT::bf16) {
2156 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2157 }
else if (VT == MVT::f32) {
2158 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
2159 }
else if (VT == MVT::f64 || (VT.
is64BitVector() && !UseLd1)) {
2160 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
2162 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
2164 if (IsPre || OffsetVal != 8)
2168 Opcode = AArch64::LD1Onev8b_POST;
2171 Opcode = AArch64::LD1Onev4h_POST;
2174 Opcode = AArch64::LD1Onev2s_POST;
2177 Opcode = AArch64::LD1Onev1d_POST;
2183 if (IsPre || OffsetVal != 16)
2187 Opcode = AArch64::LD1Onev16b_POST;
2190 Opcode = AArch64::LD1Onev8h_POST;
2193 Opcode = AArch64::LD1Onev4s_POST;
2196 Opcode = AArch64::LD1Onev2d_POST;
2207 SDValue Offset = UseLd1 ? CurDAG->getRegister(AArch64::XZR, MVT::i64)
2208 : CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
2210 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
2220 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
2221 LoadedVal =
SDValue(CurDAG->getMachineNode(AArch64::SUBREG_TO_REG, dl,
2222 MVT::i64, LoadedVal, SubReg),
2226 ReplaceUses(
SDValue(
N, 0), LoadedVal);
2229 CurDAG->RemoveDeadNode(
N);
2233void AArch64DAGToDAGISel::SelectLoad(SDNode *
N,
unsigned NumVecs,
unsigned Opc,
2234 unsigned SubRegIdx) {
2236 EVT VT =
N->getValueType(0);
2242 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2244 SDNode *Ld = CurDAG->getMachineNode(
Opc, dl, ResTys,
Ops);
2246 for (
unsigned i = 0; i < NumVecs; ++i)
2248 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
2255 MachineMemOperand *MemOp = MemIntr->getMemOperand();
2259 CurDAG->RemoveDeadNode(
N);
2262void AArch64DAGToDAGISel::SelectPostLoad(SDNode *
N,
unsigned NumVecs,
2263 unsigned Opc,
unsigned SubRegIdx) {
2265 EVT VT =
N->getValueType(0);
2272 const EVT ResTys[] = {MVT::i64,
2273 MVT::Untyped, MVT::Other};
2275 SDNode *Ld = CurDAG->getMachineNode(
Opc, dl, ResTys,
Ops);
2283 ReplaceUses(
SDValue(
N, 0), SuperReg);
2285 for (
unsigned i = 0; i < NumVecs; ++i)
2287 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
2295 CurDAG->RemoveDeadNode(
N);
2301std::tuple<unsigned, SDValue, SDValue>
2302AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *
N,
unsigned Opc_rr,
2308 SDValue NewOffset = OldOffset;
2310 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
2311 N, OldBase, NewBase, NewOffset);
2315 const bool IsRegReg =
2316 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
2319 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
2332template <SelectTypeKind Kind>
2344 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2349 if (EltVT != MVT::i1)
2353 if (EltVT == MVT::bf16)
2355 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2385void AArch64DAGToDAGISel::SelectPExtPair(SDNode *
N,
unsigned Opc) {
2388 if (
Imm->getZExtValue() > 1)
2392 EVT VT =
N->getValueType(0);
2394 SDNode *WhilePair = CurDAG->getMachineNode(
Opc,
DL, MVT::Untyped,
Ops);
2397 for (
unsigned I = 0;
I < 2; ++
I)
2398 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2399 AArch64::psub0 +
I,
DL, VT, SuperReg));
2401 CurDAG->RemoveDeadNode(
N);
2404void AArch64DAGToDAGISel::SelectWhilePair(SDNode *
N,
unsigned Opc) {
2406 EVT VT =
N->getValueType(0);
2410 SDNode *WhilePair = CurDAG->getMachineNode(
Opc,
DL, MVT::Untyped,
Ops);
2413 for (
unsigned I = 0;
I < 2; ++
I)
2414 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2415 AArch64::psub0 +
I,
DL, VT, SuperReg));
2417 CurDAG->RemoveDeadNode(
N);
2420void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *
N,
unsigned NumVecs,
2422 EVT VT =
N->getValueType(0);
2426 SDNode *
Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
Ops);
2428 for (
unsigned i = 0; i < NumVecs; ++i)
2429 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2430 AArch64::zsub0 + i,
DL, VT, SuperReg));
2432 CurDAG->RemoveDeadNode(
N);
2435void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *
N,
unsigned NumVecs,
2438 EVT VT =
N->getValueType(0);
2440 Ops.push_back(
N->getOperand(0));
2443 CurDAG->getMachineNode(Opcode,
DL, {MVT::Untyped, MVT::Other},
Ops);
2446 for (
unsigned i = 0; i < NumVecs; ++i)
2447 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2448 AArch64::zsub0 + i,
DL, VT, SuperReg));
2451 unsigned ChainIdx = NumVecs;
2453 CurDAG->RemoveDeadNode(
N);
2456void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *
N,
2461 assert(Opcode != 0 &&
"Unexpected opcode");
2464 EVT VT =
N->getValueType(0);
2465 SDUse *OpsIter =
N->op_begin() + 1;
2468 auto GetMultiVecOperand = [&]() {
2471 return createZMulTuple(Regs);
2475 Ops.push_back(*OpsIter++);
2477 Ops.push_back(GetMultiVecOperand());
2479 Ops.push_back(GetMultiVecOperand());
2481 Ops.push_back(*OpsIter++);
2484 Ops.append(OpsIter,
N->op_end());
2486 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
Ops);
2488 for (
unsigned i = 0; i < NumVecs; ++i)
2489 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2490 AArch64::zsub0 + i,
DL, VT, SuperReg));
2492 CurDAG->RemoveDeadNode(
N);
2495void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *
N,
unsigned NumVecs,
2496 unsigned Scale,
unsigned Opc_ri,
2497 unsigned Opc_rr,
bool IsIntr) {
2498 assert(Scale < 5 &&
"Invalid scaling value.");
2500 EVT VT =
N->getValueType(0);
2507 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
2508 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
2514 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2516 SDNode *
Load = CurDAG->getMachineNode(
Opc,
DL, ResTys,
Ops);
2518 for (
unsigned i = 0; i < NumVecs; ++i)
2519 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2520 AArch64::zsub0 + i,
DL, VT, SuperReg));
2523 unsigned ChainIdx = NumVecs;
2525 CurDAG->RemoveDeadNode(
N);
2528void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *
N,
2533 assert(Scale < 4 &&
"Invalid scaling value.");
2535 EVT VT =
N->getValueType(0);
2543 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
2549 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2551 SDNode *
Load = CurDAG->getMachineNode(
Opc,
DL, ResTys,
Ops);
2553 for (
unsigned i = 0; i < NumVecs; ++i)
2554 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2555 AArch64::zsub0 + i,
DL, VT, SuperReg));
2558 unsigned ChainIdx = NumVecs;
2560 CurDAG->RemoveDeadNode(
N);
2563void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *
N,
unsigned NumVecs,
2565 if (
N->getValueType(0) != MVT::nxv4f32)
2567 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
2570void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2571 unsigned NumOutVecs,
2575 if (
Imm->getZExtValue() > MaxImm)
2579 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2585 EVT VT =
Node->getValueType(0);
2588 CurDAG->getMachineNode(
Opc,
DL, {MVT::Untyped, MVT::Other},
Ops);
2591 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2592 ReplaceUses(
SDValue(Node,
I), CurDAG->getTargetExtractSubreg(
2593 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2596 unsigned ChainIdx = NumOutVecs;
2598 CurDAG->RemoveDeadNode(Node);
2601void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2602 unsigned NumIndexVecs) {
2603 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2604 "unexpected number of index vectors");
2606 constexpr unsigned FirstIndexOp = 3;
2607 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2609 if (!
Imm ||
Imm->getZExtValue() > 1)
2615 unsigned Lane =
Imm->getZExtValue();
2616 unsigned IndexOp = FirstIndexOp;
2617 if (NumIndexVecs == 3)
2620 SDValue TableTuple = createZTuple({
Node->getOperand(1),
Node->getOperand(2)});
2622 createZTuple({
Node->getOperand(IndexOp),
Node->getOperand(IndexOp + 1)});
2623 SDValue Ops[] = {TableTuple, IndexTuple,
Node->getOperand(ImmOp)};
2626 EVT VT =
Node->getValueType(0);
2628 CurDAG->getMachineNode(AArch64::LUTI6_4Z2Z2ZI,
DL, MVT::Untyped,
Ops);
2631 for (
unsigned I = 0;
I < 4; ++
I)
2632 ReplaceUses(
SDValue(Node,
I), CurDAG->getTargetExtractSubreg(
2633 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2635 CurDAG->RemoveDeadNode(Node);
2638void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2639 unsigned NumOutVecs,
2641 unsigned NumInVecs) {
2642 assert((NumInVecs == 2 || NumInVecs == 3) &&
2643 "unexpected number of input vectors");
2646 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2650 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2654 EVT VT =
Node->getValueType(0);
2657 CurDAG->getMachineNode(
Opc,
DL, {MVT::Untyped, MVT::Other},
Ops);
2660 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2661 ReplaceUses(
SDValue(Node,
I), CurDAG->getTargetExtractSubreg(
2662 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2665 CurDAG->RemoveDeadNode(Node);
2668void AArch64DAGToDAGISel::SelectClamp(SDNode *
N,
unsigned NumVecs,
2671 EVT VT =
N->getValueType(0);
2674 SDValue Zd = createZMulTuple(Regs);
2675 SDValue Zn =
N->getOperand(1 + NumVecs);
2676 SDValue Zm =
N->getOperand(2 + NumVecs);
2682 for (
unsigned i = 0; i < NumVecs; ++i)
2683 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2684 AArch64::zsub0 + i,
DL, VT, SuperReg));
2686 CurDAG->RemoveDeadNode(
N);
2716template <
unsigned MaxIdx,
unsigned Scale>
2717void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *
N,
unsigned NumVecs,
2718 unsigned BaseReg,
unsigned Op) {
2719 unsigned TileNum = 0;
2720 if (BaseReg != AArch64::ZA)
2721 TileNum =
N->getConstantOperandVal(2);
2727 if (BaseReg == AArch64::ZA)
2732 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2736 SDValue SubReg = CurDAG->getRegister(BaseReg, MVT::Other);
2738 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other},
Ops);
2740 EVT VT =
N->getValueType(0);
2741 for (
unsigned I = 0;
I < NumVecs; ++
I)
2743 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2746 unsigned ChainIdx = NumVecs;
2748 CurDAG->RemoveDeadNode(
N);
2751void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *
N,
unsigned NumVecs,
2752 unsigned Op,
unsigned MaxIdx,
2753 unsigned Scale,
unsigned BaseReg) {
2758 if (BaseReg != AArch64::ZA)
2762 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2769 if (BaseReg != AArch64::ZA )
2770 Ops.push_back(
N->getOperand(2));
2773 Ops.push_back(
N->getOperand(0));
2774 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other},
Ops);
2776 EVT VT =
N->getValueType(0);
2777 for (
unsigned I = 0;
I < NumVecs; ++
I)
2779 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2783 unsigned ChainIdx = NumVecs;
2785 CurDAG->RemoveDeadNode(
N);
2788void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *
N,
2789 unsigned NumOutVecs,
2793 EVT VT =
N->getValueType(0);
2794 unsigned NumInVecs =
N->getNumOperands() - 1;
2798 assert((NumInVecs == 2 || NumInVecs == 4) &&
2799 "Don't know how to handle multi-register input!");
2801 Ops.push_back(createZMulTuple(Regs));
2804 for (
unsigned I = 0;
I < NumInVecs;
I++)
2805 Ops.push_back(
N->getOperand(1 +
I));
2808 SDNode *Res = CurDAG->getMachineNode(
Opc,
DL, MVT::Untyped,
Ops);
2811 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2812 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2813 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2814 CurDAG->RemoveDeadNode(
N);
2817void AArch64DAGToDAGISel::SelectStore(SDNode *
N,
unsigned NumVecs,
2820 EVT VT =
N->getOperand(2)->getValueType(0);
2828 SDNode *St = CurDAG->getMachineNode(
Opc, dl,
N->getValueType(0),
Ops);
2837void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *
N,
unsigned NumVecs,
2838 unsigned Scale,
unsigned Opc_rr,
2844 SDValue RegSeq = createZTuple(Regs);
2850 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2851 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2857 SDNode *St = CurDAG->getMachineNode(
Opc, dl,
N->getValueType(0),
Ops);
2866void AArch64DAGToDAGISel::SelectPostStore(SDNode *
N,
unsigned NumVecs,
2869 EVT VT =
N->getOperand(2)->getValueType(0);
2870 const EVT ResTys[] = {MVT::i64,
2882 SDNode *St = CurDAG->getMachineNode(
Opc, dl, ResTys,
Ops);
2926void AArch64DAGToDAGISel::SelectLoadLane(SDNode *
N,
unsigned NumVecs,
2929 EVT VT =
N->getValueType(0);
2941 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2943 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2945 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2947 SDNode *Ld = CurDAG->getMachineNode(
Opc, dl, ResTys,
Ops);
2951 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2952 AArch64::qsub2, AArch64::qsub3 };
2953 for (
unsigned i = 0; i < NumVecs; ++i) {
2954 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2965 CurDAG->RemoveDeadNode(
N);
2968void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *
N,
unsigned NumVecs,
2971 EVT VT =
N->getValueType(0);
2983 const EVT ResTys[] = {MVT::i64,
2986 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2989 CurDAG->getTargetConstant(LaneNo, dl,
2994 SDNode *Ld = CurDAG->getMachineNode(
Opc, dl, ResTys,
Ops);
3006 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
3007 AArch64::qsub2, AArch64::qsub3 };
3008 for (
unsigned i = 0; i < NumVecs; ++i) {
3009 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
3023 CurDAG->RemoveDeadNode(
N);
3026void AArch64DAGToDAGISel::SelectStoreLane(SDNode *
N,
unsigned NumVecs,
3029 EVT VT =
N->getOperand(2)->getValueType(0);
3041 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
3043 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
3045 SDNode *St = CurDAG->getMachineNode(
Opc, dl, MVT::Other,
Ops);
3054void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *
N,
unsigned NumVecs,
3057 EVT VT =
N->getOperand(2)->getValueType(0);
3069 const EVT ResTys[] = {MVT::i64,
3072 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
3074 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
3078 SDNode *St = CurDAG->getMachineNode(
Opc, dl, ResTys,
Ops);
3089 unsigned &LSB,
unsigned &MSB,
3090 unsigned NumberOfIgnoredLowBits,
3091 bool BiggerPattern) {
3093 "N must be a AND operation to call this function");
3095 EVT VT =
N->getValueType(0);
3100 assert((VT == MVT::i32 || VT == MVT::i64) &&
3101 "Type checking must have been done before calling this function");
3115 const SDNode *Op0 =
N->getOperand(0).getNode();
3122 if (AndImm & (AndImm + 1))
3125 bool ClampMSB =
false;
3145 ClampMSB = (VT == MVT::i32);
3146 }
else if (BiggerPattern) {
3152 Opd0 =
N->getOperand(0);
3158 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
3161 <<
": Found large shift immediate, this should not happen\n"));
3175 MSB = MSB > 31 ? 31 : MSB;
3177 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3182 SDValue &Opd0,
unsigned &Immr,
3186 EVT VT =
N->getValueType(0);
3188 assert((VT == MVT::i32 || VT == MVT::i64) &&
3189 "Type checking must have been done before calling this function");
3193 Op =
Op->getOperand(0);
3194 VT =
Op->getValueType(0);
3203 unsigned Width =
cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3207 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
3208 Opd0 =
Op.getOperand(0);
3210 Imms = ShiftImm + Width - 1;
3215 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
3216 unsigned NumberOfIgnoredLowBits = 0,
3217 bool BiggerPattern =
false) {
3218 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
3221 switch (
N->getOpcode()) {
3223 if (!
N->isMachineOpcode())
3228 NumberOfIgnoredLowBits, BiggerPattern);
3237 unsigned NOpc =
N->getMachineOpcode();
3241 case AArch64::SBFMWri:
3242 case AArch64::UBFMWri:
3243 case AArch64::SBFMXri:
3244 case AArch64::UBFMXri:
3246 Opd0 =
N->getOperand(0);
3247 Immr =
N->getConstantOperandVal(1);
3248 Imms =
N->getConstantOperandVal(2);
3255bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *
N) {
3256 unsigned Opc, Immr, Imms;
3261 EVT VT =
N->getValueType(0);
3266 if ((
Opc == AArch64::SBFMXri ||
Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3267 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
3268 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
3270 SDNode *BFM = CurDAG->getMachineNode(
Opc, dl, MVT::i64, Ops64);
3271 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
3277 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
3278 CurDAG->getTargetConstant(Imms, dl, VT)};
3279 CurDAG->SelectNodeTo(
N,
Opc, VT,
Ops);
3288 unsigned NumberOfIgnoredHighBits,
EVT VT) {
3289 assert((VT == MVT::i32 || VT == MVT::i64) &&
3290 "i32 or i64 mask type expected!");
3294 APInt SignificantDstMask =
3298 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3299 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3332 APInt OpUsefulBits(UsefulBits);
3336 OpUsefulBits <<= MSB -
Imm + 1;
3341 OpUsefulBits <<=
Imm;
3343 OpUsefulBits <<= MSB + 1;
3352 UsefulBits &= OpUsefulBits;
3369 APInt Mask(UsefulBits);
3370 Mask.clearAllBits();
3378 Mask.lshrInPlace(ShiftAmt);
3384 Mask.lshrInPlace(ShiftAmt);
3400 APInt OpUsefulBits(UsefulBits);
3414 OpUsefulBits <<= Width;
3417 if (
Op.getOperand(1) == Orig) {
3419 Mask = ResultUsefulBits & OpUsefulBits;
3423 if (
Op.getOperand(0) == Orig)
3425 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3431 OpUsefulBits <<= Width;
3433 OpUsefulBits <<= LSB;
3435 if (
Op.getOperand(1) == Orig) {
3437 Mask = ResultUsefulBits & OpUsefulBits;
3438 Mask.lshrInPlace(LSB);
3441 if (
Op.getOperand(0) == Orig)
3442 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3459 case AArch64::ANDSWri:
3460 case AArch64::ANDSXri:
3461 case AArch64::ANDWri:
3462 case AArch64::ANDXri:
3466 case AArch64::UBFMWri:
3467 case AArch64::UBFMXri:
3470 case AArch64::ORRWrs:
3471 case AArch64::ORRXrs:
3476 case AArch64::BFMWri:
3477 case AArch64::BFMXri:
3480 case AArch64::STRBBui:
3481 case AArch64::STURBBi:
3487 case AArch64::STRHHui:
3488 case AArch64::STURHHi:
3501 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
3503 UsefulBits =
APInt(Bitwidth, 0);
3512 UsersUsefulBits |= UsefulBitsForUse;
3517 UsefulBits &= UsersUsefulBits;
3527 EVT VT =
Op.getValueType();
3530 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3533 if (ShlAmount > 0) {
3536 UBFMOpc, dl, VT,
Op,
3541 assert(ShlAmount < 0 &&
"expected right shift");
3542 int ShrAmount = -ShlAmount;
3568 bool BiggerPattern,
SDValue &Src,
3569 int &DstLSB,
int &Width) {
3570 EVT VT =
Op.getValueType();
3583 switch (
Op.getOpcode()) {
3588 NonZeroBits, Src, DstLSB, Width);
3591 NonZeroBits, Src, DstLSB, Width);
3604 EVT VT =
Op.getValueType();
3605 assert((VT == MVT::i32 || VT == MVT::i64) &&
3606 "Caller guarantees VT is one of i32 or i64");
3619 assert((~AndImm & NonZeroBits) == 0 &&
3620 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3649 if (!BiggerPattern && !AndOp0.
hasOneUse())
3668 <<
"Found large Width in bit-field-positioning -- this indicates no "
3669 "proper combining / constant folding was performed\n");
3678 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3693 "Op.getNode() should be a SHL node to call this function");
3695 "Op.getNode() should shift ShlImm to call this function");
3702 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3726 EVT VT =
Op.getValueType();
3727 assert((VT == MVT::i32 || VT == MVT::i64) &&
3728 "Caller guarantees that type is i32 or i64");
3735 if (!BiggerPattern && !
Op.hasOneUse())
3744 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3752 assert(VT == MVT::i32 || VT == MVT::i64);
3763 EVT VT =
N->getValueType(0);
3764 if (VT != MVT::i32 && VT != MVT::i64)
3782 if (!
And.hasOneUse() ||
3799 if ((OrImm & NotKnownZero) != 0) {
3811 unsigned ImmS = Width - 1;
3817 bool IsBFI = LSB != 0;
3822 unsigned OrChunks = 0, BFIChunks = 0;
3823 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3824 if (((OrImm >> Shift) & 0xFFFF) != 0)
3826 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3829 if (BFIChunks > OrChunks)
3835 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3843 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3852 if (!Dst.hasOneUse())
3855 EVT VT = Dst.getValueType();
3856 assert((VT == MVT::i32 || VT == MVT::i64) &&
3857 "Caller should guarantee that VT is one of i32 or i64");
3885 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3886 unsigned MaskWidth =
3889 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3895 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3896 ShiftedOperand =
SDValue(UBFMNode, 0);
3925 const bool BiggerPattern) {
3926 EVT VT =
N->getValueType(0);
3927 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3928 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3929 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3930 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3931 assert((VT == MVT::i32 || VT == MVT::i64) &&
3932 "Expect result type to be i32 or i64 since N is combinable to BFM");
3939 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3942 if (BiggerPattern) {
3965 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
4027 EVT VT =
N->getValueType(0);
4028 if (VT != MVT::i32 && VT != MVT::i64)
4036 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
4037 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
4057 for (
int I = 0;
I < 4; ++
I) {
4060 unsigned ImmR, ImmS;
4061 bool BiggerPattern =
I / 2;
4062 SDValue OrOpd0Val =
N->getOperand(
I % 2);
4064 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
4070 NumberOfIgnoredLowBits, BiggerPattern)) {
4073 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
4074 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
4079 Width = ImmS - ImmR + 1;
4090 Src, DstLSB, Width)) {
4098 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
4108 APInt BitsToBeInserted =
4111 if ((BitsToBeInserted & ~
Known.Zero) != 0)
4135 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4168 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4170 if (Src->hasOneUse() &&
4173 Src = Src->getOperand(0);
4183 unsigned ImmS = Width - 1;
4189 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4197bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *
N) {
4206 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
4219bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *
N) {
4223 EVT VT =
N->getValueType(0);
4224 if (VT != MVT::i32 && VT != MVT::i64)
4230 Op0, DstLSB, Width))
4236 unsigned ImmS = Width - 1;
4239 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
4240 CurDAG->getTargetConstant(ImmS,
DL, VT)};
4241 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4242 CurDAG->SelectNodeTo(
N,
Opc, VT,
Ops);
4249 bool isReciprocal) {
4252 FVal = CN->getValueAPF();
4255 if (LN->getOperand(1).getOpcode() != AArch64ISD::ADDlow ||
4265 if (
unsigned FBits =
4278 bool isReciprocal) {
4279 if ((
N.getOpcode() == AArch64ISD::NVCAST ||
N.getOpcode() ==
ISD::BITCAST) &&
4280 N.getValueType().getScalarSizeInBits() ==
4281 N.getOperand(0).getValueType().getScalarSizeInBits())
4282 N =
N.getOperand(0);
4284 auto ImmToFloat = [RegWidth](
APInt Imm) {
4298 switch (
N->getOpcode()) {
4299 case AArch64ISD::MOVIshift:
4300 FVal = ImmToFloat(
APInt(RegWidth,
N.getConstantOperandVal(0)
4301 <<
N.getConstantOperandVal(1)));
4303 case AArch64ISD::FMOV:
4304 FVal = ImmToFloat(
DecodeFMOVImm(
N.getConstantOperandVal(0), RegWidth));
4306 case AArch64ISD::DUP:
4308 FVal = ImmToFloat(
N.getConstantOperandAPInt(0).trunc(RegWidth));
4316 if (
unsigned FBits =
4325bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
4326 unsigned RegWidth) {
4331bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(
SDValue N,
SDValue &FixedPos,
4332 unsigned RegWidth) {
4334 CurDAG,
N, FixedPos, RegWidth,
false);
4337bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(
SDValue N,
4339 unsigned RegWidth) {
4341 CurDAG,
N, FixedPos, RegWidth,
true);
4344bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
4346 unsigned RegWidth) {
4356 RegString.
split(Fields,
':');
4358 if (Fields.
size() == 1)
4362 &&
"Invalid number of fields in read register string");
4365 bool AllIntFields =
true;
4369 AllIntFields &= !
Field.getAsInteger(10, IntField);
4370 Ops.push_back(IntField);
4374 "Unexpected non-integer value in special register string.");
4379 return (
Ops[0] << 14) | (
Ops[1] << 11) | (
Ops[2] << 7) | (
Ops[3] << 3) |
4387bool AArch64DAGToDAGISel::tryReadRegister(SDNode *
N) {
4389 const auto *RegString =
cast<MDString>(MD->getMD()->getOperand(0));
4392 bool ReadIs128Bit =
N->getOpcode() == AArch64ISD::MRRS;
4394 unsigned Opcode64Bit = AArch64::MRS;
4399 const auto *TheReg =
4400 AArch64SysReg::lookupSysRegByName(RegString->getString());
4401 if (TheReg && TheReg->Readable &&
4402 TheReg->haveFeatures(
Subtarget->getFeatureBits()))
4403 Imm = TheReg->Encoding;
4409 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
4410 Opcode64Bit = AArch64::ADR;
4419 RegString->getString());
4420 unsigned PseudoOp = 0;
4421 if (AArch64::GPR64RegClass.
contains(PReg))
4422 PseudoOp = AArch64::READ_REGISTER_GPR64;
4423 else if (AArch64::FPR64RegClass.
contains(PReg))
4424 PseudoOp = AArch64::READ_REGISTER_FPR64;
4425 if (!ReadIs128Bit && PseudoOp &&
N->getValueType(0) == MVT::i64) {
4426 CurDAG->SelectNodeTo(
N, PseudoOp, MVT::i64, MVT::Other,
4427 {CurDAG->getTargetConstant(PReg,
DL, MVT::i32),
4436 SDValue InChain =
N->getOperand(0);
4437 SDValue SysRegImm = CurDAG->getTargetConstant(
Imm,
DL, MVT::i32);
4438 if (!ReadIs128Bit) {
4439 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
4440 {SysRegImm, InChain});
4442 SDNode *MRRS = CurDAG->getMachineNode(
4444 {MVT::Untyped , MVT::Other },
4445 {SysRegImm, InChain});
4449 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
4451 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
4457 ReplaceUses(
SDValue(
N, 2), OutChain);
4466bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *
N) {
4468 const auto *RegString =
cast<MDString>(MD->getMD()->getOperand(0));
4471 bool WriteIs128Bit =
N->getOpcode() == AArch64ISD::MSRR;
4473 if (!WriteIs128Bit) {
4479 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
4482 "Expected a constant integer expression.");
4483 unsigned Reg = PMapper->Encoding;
4484 uint64_t Immed =
N->getConstantOperandVal(2);
4485 CurDAG->SelectNodeTo(
4486 N, State, MVT::Other, CurDAG->getTargetConstant(
Reg,
DL, MVT::i32),
4487 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
4493 if (trySelectPState(
4494 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4495 AArch64::MSRpstateImm4))
4497 if (trySelectPState(
4498 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4499 AArch64::MSRpstateImm1))
4508 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
4509 if (TheReg && TheReg->Writeable &&
4510 TheReg->haveFeatures(
Subtarget->getFeatureBits()))
4511 Imm = TheReg->Encoding;
4521 RegString->getString());
4522 bool IsGPR = AArch64::GPR64RegClass.contains(PReg);
4523 bool IsFPR = AArch64::FPR64RegClass.contains(PReg);
4524 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4525 N->getOperand(2).getValueType() == MVT::i64) {
4527 CurDAG->getCopyToReg(
N->getOperand(0),
DL, PReg,
N->getOperand(2));
4528 SDValue RegOp = CurDAG->getRegister(PReg, MVT::i64);
4529 SDNode *FakeUse = CurDAG->getMachineNode(TargetOpcode::FAKE_USE,
DL,
4530 MVT::Other, {RegOp,
Copy});
4532 CurDAG->RemoveDeadNode(
N);
4540 if (!WriteIs128Bit) {
4541 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
4542 CurDAG->getTargetConstant(
Imm,
DL, MVT::i32),
4543 N->getOperand(2), InChain);
4547 SDNode *Pair = CurDAG->getMachineNode(
4548 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
4549 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
4552 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
4554 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
4556 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
4557 CurDAG->getTargetConstant(
Imm,
DL, MVT::i32),
4565bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *
N) {
4572 if (MemTy == MVT::i8)
4573 Opcode = AArch64::CMP_SWAP_8;
4574 else if (MemTy == MVT::i16)
4575 Opcode = AArch64::CMP_SWAP_16;
4576 else if (MemTy == MVT::i32)
4577 Opcode = AArch64::CMP_SWAP_32;
4578 else if (MemTy == MVT::i64)
4579 Opcode = AArch64::CMP_SWAP_64;
4583 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4584 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4586 SDNode *CmpSwap = CurDAG->getMachineNode(
4588 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other),
Ops);
4595 CurDAG->RemoveDeadNode(
N);
4601AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO)
const {
4602 int MemoryHint = -1;
4605 return AArch64MemoryHint::NONE;
4618bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *
N,
4624 SDValue &Shift,
bool Negate) {
4631 return SelectSVEAddSubImm(SDLoc(
N), Val, VT,
Imm, Shift, Negate);
4634bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc
DL, APInt Val, MVT VT,
4643 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4650 if ((Val & ~0xff) == 0) {
4651 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4656 if ((Val & ~0xff00) == 0) {
4657 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4669bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N, MVT VT,
4693 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4694 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4701 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4702 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4706 if (Val <= 65280 && Val % 256 == 0) {
4707 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4708 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4729 int32_t ImmVal, ShiftVal;
4734 Shift = CurDAG->getTargetConstant(ShiftVal,
DL, MVT::i32);
4735 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4741 return SelectSVESignedArithImm(SDLoc(
N), CNode->getAPIntValue(),
Imm);
4745bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc
DL, APInt Val,
4748 if (ImmVal >= -128 && ImmVal < 128) {
4749 Imm = CurDAG->getSignedTargetConstant(ImmVal,
DL, MVT::i32);
4757 uint64_t ImmVal = CNode->getZExtValue();
4767 ImmVal &= 0xFFFFFFFF;
4776 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(
N), MVT::i32);
4787 ImmVal = CI->getZExtValue();
4789 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4800 Imm = CurDAG->getTargetConstant(encoding, SDLoc(
N), MVT::i64);
4813 uint64_t ImmVal = CN->getZExtValue();
4820 if (ImmVal >
High) {
4821 if (!AllowSaturation)
4826 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(
N), MVT::i32);
4833bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *
N) {
4847 const TargetLowering *
TLI = getTargetLowering();
4850 SDValue FiOp = CurDAG->getTargetFrameIndex(
4852 int TagOffset =
N->getConstantOperandVal(3);
4854 SDNode *
Out = CurDAG->getMachineNode(
4855 AArch64::TAGPstack,
DL, MVT::i64,
4856 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->
getOperand(2),
4857 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4858 ReplaceNode(
N, Out);
4862void AArch64DAGToDAGISel::SelectTagP(SDNode *
N) {
4864 "llvm.aarch64.tagp third argument must be an immediate");
4865 if (trySelectStackSlotTagP(
N))
4872 int TagOffset =
N->getConstantOperandVal(3);
4873 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4874 {
N->getOperand(1),
N->getOperand(2)});
4875 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4876 {
SDValue(N1, 0),
N->getOperand(2)});
4877 SDNode *N3 = CurDAG->getMachineNode(
4878 AArch64::ADDG,
DL, MVT::i64,
4879 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4880 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4884bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *
N) {
4888 if (
N->getConstantOperandVal(2) != 0)
4890 if (!
N->getOperand(0).isUndef())
4894 EVT VT =
N->getValueType(0);
4895 EVT InVT =
N->getOperand(1).getValueType();
4906 "Expected to insert into a packed scalable vector!");
4909 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4910 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4911 N->getOperand(1), RC));
4915bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *
N) {
4919 if (
N->getConstantOperandVal(1) != 0)
4923 EVT VT =
N->getValueType(0);
4924 EVT InVT =
N->getOperand(0).getValueType();
4935 "Expected to extract from a packed scalable vector!");
4938 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4939 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4940 N->getOperand(0), RC));
4944bool AArch64DAGToDAGISel::trySelectXAR(SDNode *
N) {
4950 EVT VT =
N->getValueType(0);
4965 if (N0.
getOpcode() != AArch64ISD::SHL_PRED ||
4968 if (N0.
getOpcode() != AArch64ISD::SHL_PRED ||
4972 auto *
TLI =
static_cast<const AArch64TargetLowering *
>(getTargetLowering());
4973 if (!
TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4981 bool IsXOROperand =
true;
4983 IsXOROperand =
false;
4989 APInt ShlAmt, ShrAmt;
4997 if (!IsXOROperand) {
4999 SDNode *MOV = CurDAG->getMachineNode(AArch64::MOVIv2d_ns,
DL, VT,
Zero);
5002 SDValue ZSub = CurDAG->getTargetConstant(AArch64::zsub,
DL, MVT::i32);
5003 SDNode *SubRegToReg =
5004 CurDAG->getMachineNode(AArch64::SUBREG_TO_REG,
DL, VT, MOVIV, ZSub);
5015 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5016 AArch64::XAR_ZZZI_D})) {
5017 CurDAG->SelectNodeTo(
N,
Opc, VT,
Ops);
5042 SVT =
Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
5052 if (N0->
getOpcode() != AArch64ISD::VSHL ||
5060 bool IsXOROperand =
true;
5062 IsXOROperand =
false;
5065 R1 =
XOR.getOperand(0);
5066 R2 =
XOR.getOperand(1);
5076 if (ShAmt + HsAmt != VTSizeInBits)
5079 if (!IsXOROperand) {
5082 CurDAG->getMachineNode(AArch64::MOVIv2d_ns,
DL, MVT::v2i64,
Zero);
5091 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, SVT), 0);
5097 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, QVT), 0);
5098 SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub,
DL, MVT::i32);
5100 R1 =
SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG,
DL, QVT,
5103 if (
R2.getValueType() == VT)
5104 R2 =
SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG,
DL, QVT,
5109 SDValue SubReg = CurDAG->getTargetConstant(
5112 R1 =
SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG,
DL, SVT,
Undef,
5117 R2 =
SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG,
DL, SVT,
5123 SDNode *XAR =
nullptr;
5127 SVT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5128 AArch64::XAR_ZZZI_D}))
5129 XAR = CurDAG->getMachineNode(
Opc,
DL, SVT,
Ops);
5131 XAR = CurDAG->getMachineNode(AArch64::XAR,
DL, SVT,
Ops);
5134 assert(XAR &&
"Unexpected NULL value for XAR instruction in DAG");
5140 SDValue ZSub = CurDAG->getTargetConstant(AArch64::zsub,
DL, MVT::i32);
5141 SDNode *Q = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG,
DL, QVT,
5144 SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub,
DL, MVT::i32);
5145 XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG,
DL, VT,
5148 SDValue SubReg = CurDAG->getTargetConstant(
5151 XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG,
DL, VT,
5155 ReplaceNode(
N, XAR);
5162 assert(VT == MVT::i32 || VT == MVT::i64);
5164 VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5167void AArch64DAGToDAGISel::Select(SDNode *Node) {
5169 if (
Node->isMachineOpcode()) {
5171 Node->setNodeId(-1);
5176 EVT VT =
Node->getValueType(0);
5178 switch (
Node->getOpcode()) {
5183 if (SelectCMP_SWAP(Node))
5188 case AArch64ISD::MRRS:
5189 if (tryReadRegister(Node))
5194 case AArch64ISD::MSRR:
5195 if (tryWriteRegister(Node))
5202 if (tryIndexedLoad(Node))
5211 if (tryBitfieldExtractOp(Node))
5213 if (tryBitfieldInsertInZeroOp(Node))
5221 if (tryBitfieldInsertOp(Node))
5223 if (trySelectXAR(Node))
5228 if (trySelectCastScalableToFixedLengthVector(Node))
5234 if (trySelectCastFixedLengthToScalableVector(Node))
5239 case AArch64ISD::CSEL:
5240 if (tryFoldCselToFMaxMin(Node))
5248 if (ConstNode->
isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5259 const TargetLowering *
TLI = getTargetLowering();
5260 SDValue TFI = CurDAG->getTargetFrameIndex(
5263 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
5264 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
5265 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64,
Ops);
5269 unsigned IntNo =
Node->getConstantOperandVal(1);
5273 case Intrinsic::aarch64_gcsss: {
5277 SDValue Zero = CurDAG->getCopyFromReg(Chain,
DL, AArch64::XZR, MVT::i64);
5279 CurDAG->getMachineNode(AArch64::GCSSS1,
DL, MVT::Other, Val, Chain);
5280 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2,
DL, MVT::i64,
5282 ReplaceNode(Node, SS2);
5285 case Intrinsic::aarch64_ldaxp:
5286 case Intrinsic::aarch64_ldxp: {
5288 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5293 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
5294 MVT::Other, MemAddr, Chain);
5297 MachineMemOperand *MemOp =
5300 ReplaceNode(Node, Ld);
5303 case Intrinsic::aarch64_stlxp:
5304 case Intrinsic::aarch64_stxp: {
5306 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5314 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5316 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other,
Ops);
5318 MachineMemOperand *MemOp =
5322 ReplaceNode(Node, St);
5325 case Intrinsic::aarch64_neon_ld1x2:
5326 if (VT == MVT::v8i8) {
5327 SelectLoad(Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
5329 }
else if (VT == MVT::v16i8) {
5330 SelectLoad(Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
5332 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5333 SelectLoad(Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
5335 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5336 SelectLoad(Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
5338 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5339 SelectLoad(Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
5341 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5342 SelectLoad(Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
5344 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5345 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
5347 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5348 SelectLoad(Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
5352 case Intrinsic::aarch64_neon_ld1x3:
5353 if (VT == MVT::v8i8) {
5354 SelectLoad(Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
5356 }
else if (VT == MVT::v16i8) {
5357 SelectLoad(Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
5359 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5360 SelectLoad(Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
5362 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5363 SelectLoad(Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
5365 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5366 SelectLoad(Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
5368 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5369 SelectLoad(Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
5371 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5372 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
5374 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5375 SelectLoad(Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
5379 case Intrinsic::aarch64_neon_ld1x4:
5380 if (VT == MVT::v8i8) {
5381 SelectLoad(Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
5383 }
else if (VT == MVT::v16i8) {
5384 SelectLoad(Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
5386 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5387 SelectLoad(Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
5389 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5390 SelectLoad(Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
5392 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5393 SelectLoad(Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
5395 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5396 SelectLoad(Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
5398 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5399 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
5401 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5402 SelectLoad(Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
5406 case Intrinsic::aarch64_neon_ld2:
5407 if (VT == MVT::v8i8) {
5408 SelectLoad(Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
5410 }
else if (VT == MVT::v16i8) {
5411 SelectLoad(Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
5413 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5414 SelectLoad(Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
5416 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5417 SelectLoad(Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
5419 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5420 SelectLoad(Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
5422 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5423 SelectLoad(Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
5425 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5426 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
5428 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5429 SelectLoad(Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
5433 case Intrinsic::aarch64_neon_ld3:
5434 if (VT == MVT::v8i8) {
5435 SelectLoad(Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
5437 }
else if (VT == MVT::v16i8) {
5438 SelectLoad(Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
5440 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5441 SelectLoad(Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
5443 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5444 SelectLoad(Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
5446 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5447 SelectLoad(Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
5449 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5450 SelectLoad(Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
5452 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5453 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
5455 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5456 SelectLoad(Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
5460 case Intrinsic::aarch64_neon_ld4:
5461 if (VT == MVT::v8i8) {
5462 SelectLoad(Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
5464 }
else if (VT == MVT::v16i8) {
5465 SelectLoad(Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
5467 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5468 SelectLoad(Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
5470 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5471 SelectLoad(Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
5473 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5474 SelectLoad(Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
5476 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5477 SelectLoad(Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
5479 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5480 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
5482 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5483 SelectLoad(Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
5487 case Intrinsic::aarch64_neon_ld2r:
5488 if (VT == MVT::v8i8) {
5489 SelectLoad(Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
5491 }
else if (VT == MVT::v16i8) {
5492 SelectLoad(Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
5494 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5495 SelectLoad(Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
5497 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5498 SelectLoad(Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
5500 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5501 SelectLoad(Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
5503 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5504 SelectLoad(Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
5506 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5507 SelectLoad(Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
5509 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5510 SelectLoad(Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
5514 case Intrinsic::aarch64_neon_ld3r:
5515 if (VT == MVT::v8i8) {
5516 SelectLoad(Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
5518 }
else if (VT == MVT::v16i8) {
5519 SelectLoad(Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
5521 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5522 SelectLoad(Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
5524 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5525 SelectLoad(Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
5527 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5528 SelectLoad(Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
5530 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5531 SelectLoad(Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
5533 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5534 SelectLoad(Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
5536 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5537 SelectLoad(Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
5541 case Intrinsic::aarch64_neon_ld4r:
5542 if (VT == MVT::v8i8) {
5543 SelectLoad(Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
5545 }
else if (VT == MVT::v16i8) {
5546 SelectLoad(Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
5548 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5549 SelectLoad(Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
5551 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5552 SelectLoad(Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
5554 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5555 SelectLoad(Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
5557 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5558 SelectLoad(Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
5560 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5561 SelectLoad(Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
5563 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5564 SelectLoad(Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
5568 case Intrinsic::aarch64_neon_ld2lane:
5569 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5570 SelectLoadLane(Node, 2, AArch64::LD2i8);
5572 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5573 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5574 SelectLoadLane(Node, 2, AArch64::LD2i16);
5576 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5578 SelectLoadLane(Node, 2, AArch64::LD2i32);
5580 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5582 SelectLoadLane(Node, 2, AArch64::LD2i64);
5586 case Intrinsic::aarch64_neon_ld3lane:
5587 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5588 SelectLoadLane(Node, 3, AArch64::LD3i8);
5590 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5591 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5592 SelectLoadLane(Node, 3, AArch64::LD3i16);
5594 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5596 SelectLoadLane(Node, 3, AArch64::LD3i32);
5598 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5600 SelectLoadLane(Node, 3, AArch64::LD3i64);
5604 case Intrinsic::aarch64_neon_ld4lane:
5605 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5606 SelectLoadLane(Node, 4, AArch64::LD4i8);
5608 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5609 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5610 SelectLoadLane(Node, 4, AArch64::LD4i16);
5612 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5614 SelectLoadLane(Node, 4, AArch64::LD4i32);
5616 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5618 SelectLoadLane(Node, 4, AArch64::LD4i64);
5622 case Intrinsic::aarch64_ld64b:
5623 SelectLoad(Node, 8, AArch64::LD64B, AArch64::x8sub_0);
5625 case Intrinsic::aarch64_sve_ld2q_sret: {
5626 SelectPredicatedLoad(Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
5629 case Intrinsic::aarch64_sve_ld3q_sret: {
5630 SelectPredicatedLoad(Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
5633 case Intrinsic::aarch64_sve_ld4q_sret: {
5634 SelectPredicatedLoad(Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
5637 case Intrinsic::aarch64_sve_ld2_sret: {
5638 if (VT == MVT::nxv16i8) {
5639 SelectPredicatedLoad(Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
5642 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5643 VT == MVT::nxv8bf16) {
5644 SelectPredicatedLoad(Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
5647 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5648 SelectPredicatedLoad(Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
5651 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5652 SelectPredicatedLoad(Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
5658 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5659 if (VT == MVT::nxv16i8) {
5661 SelectContiguousMultiVectorLoad(
5662 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
5664 SelectContiguousMultiVectorLoad(Node, 2, 0, AArch64::LD1B_2Z_IMM,
5669 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5670 VT == MVT::nxv8bf16) {
5672 SelectContiguousMultiVectorLoad(
5673 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
5675 SelectContiguousMultiVectorLoad(Node, 2, 1, AArch64::LD1H_2Z_IMM,
5680 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5682 SelectContiguousMultiVectorLoad(
5683 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
5685 SelectContiguousMultiVectorLoad(Node, 2, 2, AArch64::LD1W_2Z_IMM,
5690 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5692 SelectContiguousMultiVectorLoad(
5693 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
5695 SelectContiguousMultiVectorLoad(Node, 2, 3, AArch64::LD1D_2Z_IMM,
5703 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5704 if (VT == MVT::nxv16i8) {
5706 SelectContiguousMultiVectorLoad(
5707 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5709 SelectContiguousMultiVectorLoad(Node, 4, 0, AArch64::LD1B_4Z_IMM,
5714 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5715 VT == MVT::nxv8bf16) {
5717 SelectContiguousMultiVectorLoad(
5718 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5720 SelectContiguousMultiVectorLoad(Node, 4, 1, AArch64::LD1H_4Z_IMM,
5725 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5727 SelectContiguousMultiVectorLoad(
5728 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5730 SelectContiguousMultiVectorLoad(Node, 4, 2, AArch64::LD1W_4Z_IMM,
5735 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5737 SelectContiguousMultiVectorLoad(
5738 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5740 SelectContiguousMultiVectorLoad(Node, 4, 3, AArch64::LD1D_4Z_IMM,
5748 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5749 if (VT == MVT::nxv16i8) {
5751 SelectContiguousMultiVectorLoad(Node, 2, 0,
5752 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5753 AArch64::LDNT1B_2Z_PSEUDO);
5755 SelectContiguousMultiVectorLoad(Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5756 AArch64::LDNT1B_2Z);
5760 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5761 VT == MVT::nxv8bf16) {
5763 SelectContiguousMultiVectorLoad(Node, 2, 1,
5764 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5765 AArch64::LDNT1H_2Z_PSEUDO);
5767 SelectContiguousMultiVectorLoad(Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5768 AArch64::LDNT1H_2Z);
5772 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5774 SelectContiguousMultiVectorLoad(Node, 2, 2,
5775 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5776 AArch64::LDNT1W_2Z_PSEUDO);
5778 SelectContiguousMultiVectorLoad(Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5779 AArch64::LDNT1W_2Z);
5783 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5785 SelectContiguousMultiVectorLoad(Node, 2, 3,
5786 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5787 AArch64::LDNT1D_2Z_PSEUDO);
5789 SelectContiguousMultiVectorLoad(Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5790 AArch64::LDNT1D_2Z);
5797 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5798 if (VT == MVT::nxv16i8) {
5800 SelectContiguousMultiVectorLoad(Node, 4, 0,
5801 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5802 AArch64::LDNT1B_4Z_PSEUDO);
5804 SelectContiguousMultiVectorLoad(Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5805 AArch64::LDNT1B_4Z);
5809 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5810 VT == MVT::nxv8bf16) {
5812 SelectContiguousMultiVectorLoad(Node, 4, 1,
5813 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5814 AArch64::LDNT1H_4Z_PSEUDO);
5816 SelectContiguousMultiVectorLoad(Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5817 AArch64::LDNT1H_4Z);
5821 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5823 SelectContiguousMultiVectorLoad(Node, 4, 2,
5824 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5825 AArch64::LDNT1W_4Z_PSEUDO);
5827 SelectContiguousMultiVectorLoad(Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5828 AArch64::LDNT1W_4Z);
5832 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5834 SelectContiguousMultiVectorLoad(Node, 4, 3,
5835 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5836 AArch64::LDNT1D_4Z_PSEUDO);
5838 SelectContiguousMultiVectorLoad(Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5839 AArch64::LDNT1D_4Z);
5846 case Intrinsic::aarch64_sve_ld3_sret: {
5847 if (VT == MVT::nxv16i8) {
5848 SelectPredicatedLoad(Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5851 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5852 VT == MVT::nxv8bf16) {
5853 SelectPredicatedLoad(Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5856 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5857 SelectPredicatedLoad(Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5860 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5861 SelectPredicatedLoad(Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5867 case Intrinsic::aarch64_sve_ld4_sret: {
5868 if (VT == MVT::nxv16i8) {
5869 SelectPredicatedLoad(Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5872 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5873 VT == MVT::nxv8bf16) {
5874 SelectPredicatedLoad(Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5877 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5878 SelectPredicatedLoad(Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5881 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5882 SelectPredicatedLoad(Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5888 case Intrinsic::aarch64_sme_read_hor_vg2: {
5889 if (VT == MVT::nxv16i8) {
5890 SelectMultiVectorMove<14, 2>(Node, 2, AArch64::ZAB0,
5891 AArch64::MOVA_2ZMXI_H_B);
5893 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5894 VT == MVT::nxv8bf16) {
5895 SelectMultiVectorMove<6, 2>(Node, 2, AArch64::ZAH0,
5896 AArch64::MOVA_2ZMXI_H_H);
5898 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5899 SelectMultiVectorMove<2, 2>(Node, 2, AArch64::ZAS0,
5900 AArch64::MOVA_2ZMXI_H_S);
5902 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5903 SelectMultiVectorMove<0, 2>(Node, 2, AArch64::ZAD0,
5904 AArch64::MOVA_2ZMXI_H_D);
5909 case Intrinsic::aarch64_sme_read_ver_vg2: {
5910 if (VT == MVT::nxv16i8) {
5911 SelectMultiVectorMove<14, 2>(Node, 2, AArch64::ZAB0,
5912 AArch64::MOVA_2ZMXI_V_B);
5914 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5915 VT == MVT::nxv8bf16) {
5916 SelectMultiVectorMove<6, 2>(Node, 2, AArch64::ZAH0,
5917 AArch64::MOVA_2ZMXI_V_H);
5919 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5920 SelectMultiVectorMove<2, 2>(Node, 2, AArch64::ZAS0,
5921 AArch64::MOVA_2ZMXI_V_S);
5923 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5924 SelectMultiVectorMove<0, 2>(Node, 2, AArch64::ZAD0,
5925 AArch64::MOVA_2ZMXI_V_D);
5930 case Intrinsic::aarch64_sme_read_hor_vg4: {
5931 if (VT == MVT::nxv16i8) {
5932 SelectMultiVectorMove<12, 4>(Node, 4, AArch64::ZAB0,
5933 AArch64::MOVA_4ZMXI_H_B);
5935 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5936 VT == MVT::nxv8bf16) {
5937 SelectMultiVectorMove<4, 4>(Node, 4, AArch64::ZAH0,
5938 AArch64::MOVA_4ZMXI_H_H);
5940 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5941 SelectMultiVectorMove<0, 2>(Node, 4, AArch64::ZAS0,
5942 AArch64::MOVA_4ZMXI_H_S);
5944 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5945 SelectMultiVectorMove<0, 2>(Node, 4, AArch64::ZAD0,
5946 AArch64::MOVA_4ZMXI_H_D);
5951 case Intrinsic::aarch64_sme_read_ver_vg4: {
5952 if (VT == MVT::nxv16i8) {
5953 SelectMultiVectorMove<12, 4>(Node, 4, AArch64::ZAB0,
5954 AArch64::MOVA_4ZMXI_V_B);
5956 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5957 VT == MVT::nxv8bf16) {
5958 SelectMultiVectorMove<4, 4>(Node, 4, AArch64::ZAH0,
5959 AArch64::MOVA_4ZMXI_V_H);
5961 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5962 SelectMultiVectorMove<0, 4>(Node, 4, AArch64::ZAS0,
5963 AArch64::MOVA_4ZMXI_V_S);
5965 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5966 SelectMultiVectorMove<0, 4>(Node, 4, AArch64::ZAD0,
5967 AArch64::MOVA_4ZMXI_V_D);
5972 case Intrinsic::aarch64_sme_read_vg1x2: {
5973 SelectMultiVectorMove<7, 1>(Node, 2, AArch64::ZA,
5974 AArch64::MOVA_VG2_2ZMXI);
5977 case Intrinsic::aarch64_sme_read_vg1x4: {
5978 SelectMultiVectorMove<7, 1>(Node, 4, AArch64::ZA,
5979 AArch64::MOVA_VG4_4ZMXI);
5982 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5983 if (VT == MVT::nxv16i8) {
5984 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5986 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5987 VT == MVT::nxv8bf16) {
5988 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5990 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5991 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5993 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5994 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5999 case Intrinsic::aarch64_sme_readz_vert_x2: {
6000 if (VT == MVT::nxv16i8) {
6001 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
6003 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6004 VT == MVT::nxv8bf16) {
6005 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
6007 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6008 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
6010 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6011 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
6016 case Intrinsic::aarch64_sme_readz_horiz_x4: {
6017 if (VT == MVT::nxv16i8) {
6018 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
6020 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6021 VT == MVT::nxv8bf16) {
6022 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
6024 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6025 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
6027 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6028 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
6033 case Intrinsic::aarch64_sme_readz_vert_x4: {
6034 if (VT == MVT::nxv16i8) {
6035 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
6037 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6038 VT == MVT::nxv8bf16) {
6039 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
6041 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6042 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
6044 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6045 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
6050 case Intrinsic::aarch64_sme_readz_x2: {
6051 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, 7, 1,
6055 case Intrinsic::aarch64_sme_readz_x4: {
6056 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, 7, 1,
6060 case Intrinsic::swift_async_context_addr: {
6063 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
6065 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
6066 CurDAG->getTargetConstant(8,
DL, MVT::i32),
6067 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
6069 ReplaceUses(
SDValue(Node, 0), Res);
6071 CurDAG->RemoveDeadNode(Node);
6073 auto &MF = CurDAG->getMachineFunction();
6074 MF.getFrameInfo().setFrameAddressIsTaken(
true);
6075 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(
true);
6078 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6080 Node->getValueType(0),
6081 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6082 AArch64::LUTI2_4ZTZI_S}))
6084 SelectMultiVectorLutiLane(Node, 4,
Opc, 3);
6087 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6089 Node->getValueType(0),
6090 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6092 SelectMultiVectorLutiLane(Node, 4,
Opc, 1);
6095 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6097 Node->getValueType(0),
6098 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6099 AArch64::LUTI2_2ZTZI_S}))
6101 SelectMultiVectorLutiLane(Node, 2,
Opc, 7);
6104 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6106 Node->getValueType(0),
6107 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6108 AArch64::LUTI4_2ZTZI_S}))
6110 SelectMultiVectorLutiLane(Node, 2,
Opc, 3);
6113 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6114 SelectMultiVectorLuti(Node, 4, AArch64::LUTI4_4ZZT2Z, 2);
6117 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6118 SelectMultiVectorLuti(Node, 4, AArch64::LUTI6_4ZT3Z, 3);
6121 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6123 Node->getValueType(0),
6124 {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6125 SelectCVTIntrinsicFP8(Node, 2,
Opc);
6127 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6129 Node->getValueType(0),
6130 {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6131 SelectCVTIntrinsicFP8(Node, 2,
Opc);
6133 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6135 Node->getValueType(0),
6136 {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6137 SelectCVTIntrinsicFP8(Node, 2,
Opc);
6139 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6141 Node->getValueType(0),
6142 {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6143 SelectCVTIntrinsicFP8(Node, 2,
Opc);
6145 case Intrinsic::ptrauth_resign_load_relative:
6146 SelectPtrauthResign(Node);
6151 unsigned IntNo =
Node->getConstantOperandVal(0);
6155 case Intrinsic::aarch64_tagp:
6159 case Intrinsic::ptrauth_auth:
6160 SelectPtrauthAuth(Node);
6163 case Intrinsic::ptrauth_resign:
6164 SelectPtrauthResign(Node);
6167 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6168 SelectPtrauthResignWithPC(Node);
6171 case Intrinsic::aarch64_neon_tbl2:
6172 SelectTable(Node, 2,
6173 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6176 case Intrinsic::aarch64_neon_tbl3:
6177 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6178 : AArch64::TBLv16i8Three,
6181 case Intrinsic::aarch64_neon_tbl4:
6182 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6183 : AArch64::TBLv16i8Four,
6186 case Intrinsic::aarch64_neon_tbx2:
6187 SelectTable(Node, 2,
6188 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6191 case Intrinsic::aarch64_neon_tbx3:
6192 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6193 : AArch64::TBXv16i8Three,
6196 case Intrinsic::aarch64_neon_tbx4:
6197 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6198 : AArch64::TBXv16i8Four,
6201 case Intrinsic::aarch64_sve_srshl_single_x2:
6203 Node->getValueType(0),
6204 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6205 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6206 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6208 case Intrinsic::aarch64_sve_srshl_single_x4:
6210 Node->getValueType(0),
6211 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6212 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6213 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6215 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6216 SelectMultiVectorLuti6LaneX4(Node, 2);
6218 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6219 SelectMultiVectorLuti6LaneX4(Node, 3);
6221 case Intrinsic::aarch64_sve_urshl_single_x2:
6223 Node->getValueType(0),
6224 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6225 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6226 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6228 case Intrinsic::aarch64_sve_urshl_single_x4:
6230 Node->getValueType(0),
6231 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6232 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6233 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6235 case Intrinsic::aarch64_sve_srshl_x2:
6237 Node->getValueType(0),
6238 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6239 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6240 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6242 case Intrinsic::aarch64_sve_srshl_x4:
6244 Node->getValueType(0),
6245 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6246 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6247 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6249 case Intrinsic::aarch64_sve_urshl_x2:
6251 Node->getValueType(0),
6252 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6253 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6254 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6256 case Intrinsic::aarch64_sve_urshl_x4:
6258 Node->getValueType(0),
6259 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6260 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6261 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6263 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6265 Node->getValueType(0),
6266 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6267 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6268 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6270 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6272 Node->getValueType(0),
6273 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6274 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6275 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6277 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6279 Node->getValueType(0),
6280 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6281 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6282 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6284 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6286 Node->getValueType(0),
6287 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6288 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6289 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6291 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6293 Node->getValueType(0),
6294 {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6295 AArch64::FSCALE_2ZZ_D}))
6296 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6298 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6300 Node->getValueType(0),
6301 {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6302 AArch64::FSCALE_4ZZ_D}))
6303 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6305 case Intrinsic::aarch64_sme_fp8_scale_x2:
6307 Node->getValueType(0),
6308 {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6309 AArch64::FSCALE_2Z2Z_D}))
6310 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6312 case Intrinsic::aarch64_sme_fp8_scale_x4:
6314 Node->getValueType(0),
6315 {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6316 AArch64::FSCALE_4Z4Z_D}))
6317 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6319 case Intrinsic::aarch64_sve_whilege_x2:
6321 Node->getValueType(0),
6322 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6323 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6324 SelectWhilePair(Node,
Op);
6326 case Intrinsic::aarch64_sve_whilegt_x2:
6328 Node->getValueType(0),
6329 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6330 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6331 SelectWhilePair(Node,
Op);
6333 case Intrinsic::aarch64_sve_whilehi_x2:
6335 Node->getValueType(0),
6336 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6337 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6338 SelectWhilePair(Node,
Op);
6340 case Intrinsic::aarch64_sve_whilehs_x2:
6342 Node->getValueType(0),
6343 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6344 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6345 SelectWhilePair(Node,
Op);
6347 case Intrinsic::aarch64_sve_whilele_x2:
6349 Node->getValueType(0),
6350 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6351 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6352 SelectWhilePair(Node,
Op);
6354 case Intrinsic::aarch64_sve_whilelo_x2:
6356 Node->getValueType(0),
6357 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6358 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6359 SelectWhilePair(Node,
Op);
6361 case Intrinsic::aarch64_sve_whilels_x2:
6363 Node->getValueType(0),
6364 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6365 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6366 SelectWhilePair(Node,
Op);
6368 case Intrinsic::aarch64_sve_whilelt_x2:
6370 Node->getValueType(0),
6371 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6372 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6373 SelectWhilePair(Node,
Op);
6375 case Intrinsic::aarch64_sve_smax_single_x2:
6377 Node->getValueType(0),
6378 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6379 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6380 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6382 case Intrinsic::aarch64_sve_umax_single_x2:
6384 Node->getValueType(0),
6385 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6386 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6387 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6389 case Intrinsic::aarch64_sve_fmax_single_x2:
6391 Node->getValueType(0),
6392 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6393 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6394 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6396 case Intrinsic::aarch64_sve_smax_single_x4:
6398 Node->getValueType(0),
6399 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6400 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6401 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6403 case Intrinsic::aarch64_sve_umax_single_x4:
6405 Node->getValueType(0),
6406 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6407 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6408 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6410 case Intrinsic::aarch64_sve_fmax_single_x4:
6412 Node->getValueType(0),
6413 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6414 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6415 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6417 case Intrinsic::aarch64_sve_smin_single_x2:
6419 Node->getValueType(0),
6420 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6421 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6422 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6424 case Intrinsic::aarch64_sve_umin_single_x2:
6426 Node->getValueType(0),
6427 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6428 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6429 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6431 case Intrinsic::aarch64_sve_fmin_single_x2:
6433 Node->getValueType(0),
6434 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6435 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6436 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6438 case Intrinsic::aarch64_sve_smin_single_x4:
6440 Node->getValueType(0),
6441 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6442 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6443 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6445 case Intrinsic::aarch64_sve_umin_single_x4:
6447 Node->getValueType(0),
6448 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6449 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6450 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6452 case Intrinsic::aarch64_sve_fmin_single_x4:
6454 Node->getValueType(0),
6455 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6456 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6457 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6459 case Intrinsic::aarch64_sve_smax_x2:
6461 Node->getValueType(0),
6462 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6463 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6464 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6466 case Intrinsic::aarch64_sve_umax_x2:
6468 Node->getValueType(0),
6469 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6470 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6471 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6473 case Intrinsic::aarch64_sve_fmax_x2:
6475 Node->getValueType(0),
6476 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6477 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6478 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6480 case Intrinsic::aarch64_sve_smax_x4:
6482 Node->getValueType(0),
6483 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6484 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6485 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6487 case Intrinsic::aarch64_sve_umax_x4:
6489 Node->getValueType(0),
6490 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6491 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6492 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6494 case Intrinsic::aarch64_sve_fmax_x4:
6496 Node->getValueType(0),
6497 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6498 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6499 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6501 case Intrinsic::aarch64_sme_famax_x2:
6503 Node->getValueType(0),
6504 {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6505 AArch64::FAMAX_2Z2Z_D}))
6506 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6508 case Intrinsic::aarch64_sme_famax_x4:
6510 Node->getValueType(0),
6511 {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6512 AArch64::FAMAX_4Z4Z_D}))
6513 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6515 case Intrinsic::aarch64_sme_famin_x2:
6517 Node->getValueType(0),
6518 {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6519 AArch64::FAMIN_2Z2Z_D}))
6520 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6522 case Intrinsic::aarch64_sme_famin_x4:
6524 Node->getValueType(0),
6525 {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6526 AArch64::FAMIN_4Z4Z_D}))
6527 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6529 case Intrinsic::aarch64_sve_smin_x2:
6531 Node->getValueType(0),
6532 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6533 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6534 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6536 case Intrinsic::aarch64_sve_umin_x2:
6538 Node->getValueType(0),
6539 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6540 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6541 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6543 case Intrinsic::aarch64_sve_fmin_x2:
6545 Node->getValueType(0),
6546 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6547 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6548 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6550 case Intrinsic::aarch64_sve_smin_x4:
6552 Node->getValueType(0),
6553 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6554 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6555 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6557 case Intrinsic::aarch64_sve_umin_x4:
6559 Node->getValueType(0),
6560 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6561 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6562 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6564 case Intrinsic::aarch64_sve_fmin_x4:
6566 Node->getValueType(0),
6567 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6568 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6569 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6571 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6573 Node->getValueType(0),
6574 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6575 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6576 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6578 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6580 Node->getValueType(0),
6581 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6582 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6583 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6585 case Intrinsic::aarch64_sve_fminnm_single_x2:
6587 Node->getValueType(0),
6588 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6589 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6590 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6592 case Intrinsic::aarch64_sve_fminnm_single_x4:
6594 Node->getValueType(0),
6595 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6596 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6597 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6599 case Intrinsic::aarch64_sve_fscale_single_x4:
6600 SelectDestructiveMultiIntrinsic(Node, 4,
false, AArch64::BFSCALE_4ZZ);
6602 case Intrinsic::aarch64_sve_fscale_single_x2:
6603 SelectDestructiveMultiIntrinsic(Node, 2,
false, AArch64::BFSCALE_2ZZ);
6605 case Intrinsic::aarch64_sve_fmul_single_x4:
6607 Node->getValueType(0),
6608 {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6609 AArch64::FMUL_4ZZ_D}))
6610 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6612 case Intrinsic::aarch64_sve_fmul_single_x2:
6614 Node->getValueType(0),
6615 {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6616 AArch64::FMUL_2ZZ_D}))
6617 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6619 case Intrinsic::aarch64_sve_fmaxnm_x2:
6621 Node->getValueType(0),
6622 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6623 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6624 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6626 case Intrinsic::aarch64_sve_fmaxnm_x4:
6628 Node->getValueType(0),
6629 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6630 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6631 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6633 case Intrinsic::aarch64_sve_fminnm_x2:
6635 Node->getValueType(0),
6636 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6637 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6638 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6640 case Intrinsic::aarch64_sve_fminnm_x4:
6642 Node->getValueType(0),
6643 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6644 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6645 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6647 case Intrinsic::aarch64_sve_aese_lane_x2:
6648 SelectDestructiveMultiIntrinsic(Node, 2,
false, AArch64::AESE_2ZZI_B);
6650 case Intrinsic::aarch64_sve_aesd_lane_x2:
6651 SelectDestructiveMultiIntrinsic(Node, 2,
false, AArch64::AESD_2ZZI_B);
6653 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6654 SelectDestructiveMultiIntrinsic(Node, 2,
false, AArch64::AESEMC_2ZZI_B);
6656 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6657 SelectDestructiveMultiIntrinsic(Node, 2,
false, AArch64::AESDIMC_2ZZI_B);
6659 case Intrinsic::aarch64_sve_aese_lane_x4:
6660 SelectDestructiveMultiIntrinsic(Node, 4,
false, AArch64::AESE_4ZZI_B);
6662 case Intrinsic::aarch64_sve_aesd_lane_x4:
6663 SelectDestructiveMultiIntrinsic(Node, 4,
false, AArch64::AESD_4ZZI_B);
6665 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6666 SelectDestructiveMultiIntrinsic(Node, 4,
false, AArch64::AESEMC_4ZZI_B);
6668 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6669 SelectDestructiveMultiIntrinsic(Node, 4,
false, AArch64::AESDIMC_4ZZI_B);
6671 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6672 SelectDestructiveMultiIntrinsic(Node, 2,
false, AArch64::PMLAL_2ZZZ_Q);
6674 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6678 CurDAG->getMachineNode(AArch64::PMULL_2ZZZ_Q,
DL, MVT::Untyped, Regs);
6680 for (
unsigned I = 0;
I < 2;
I++)
6682 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
6684 CurDAG->RemoveDeadNode(Node);
6687 case Intrinsic::aarch64_sve_fscale_x4:
6688 SelectDestructiveMultiIntrinsic(Node, 4,
true, AArch64::BFSCALE_4Z4Z);
6690 case Intrinsic::aarch64_sve_fscale_x2:
6691 SelectDestructiveMultiIntrinsic(Node, 2,
true, AArch64::BFSCALE_2Z2Z);
6693 case Intrinsic::aarch64_sve_fmul_x4:
6695 Node->getValueType(0),
6696 {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6697 AArch64::FMUL_4Z4Z_D}))
6698 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op);
6700 case Intrinsic::aarch64_sve_fmul_x2:
6702 Node->getValueType(0),
6703 {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6704 AArch64::FMUL_2Z2Z_D}))
6705 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op);
6707 case Intrinsic::aarch64_sve_fcvtzs_x2:
6708 SelectCVTIntrinsic(Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
6710 case Intrinsic::aarch64_sve_scvtf_x2:
6711 SelectCVTIntrinsic(Node, 2, AArch64::SCVTF_2Z2Z_StoS);
6713 case Intrinsic::aarch64_sve_fcvtzu_x2:
6714 SelectCVTIntrinsic(Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
6716 case Intrinsic::aarch64_sve_ucvtf_x2:
6717 SelectCVTIntrinsic(Node, 2, AArch64::UCVTF_2Z2Z_StoS);
6719 case Intrinsic::aarch64_sve_fcvtzs_x4:
6720 SelectCVTIntrinsic(Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
6722 case Intrinsic::aarch64_sve_scvtf_x4:
6723 SelectCVTIntrinsic(Node, 4, AArch64::SCVTF_4Z4Z_StoS);
6725 case Intrinsic::aarch64_sve_fcvtzu_x4:
6726 SelectCVTIntrinsic(Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
6728 case Intrinsic::aarch64_sve_ucvtf_x4:
6729 SelectCVTIntrinsic(Node, 4, AArch64::UCVTF_4Z4Z_StoS);
6731 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6732 SelectUnaryMultiIntrinsic(Node, 2,
false, AArch64::FCVT_2ZZ_H_S);
6734 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6735 SelectUnaryMultiIntrinsic(Node, 2,
false, AArch64::FCVTL_2ZZ_H_S);
6737 case Intrinsic::aarch64_sve_sclamp_single_x2:
6739 Node->getValueType(0),
6740 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6741 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6742 SelectClamp(Node, 2,
Op);
6744 case Intrinsic::aarch64_sve_uclamp_single_x2:
6746 Node->getValueType(0),
6747 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6748 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6749 SelectClamp(Node, 2,
Op);
6751 case Intrinsic::aarch64_sve_fclamp_single_x2:
6753 Node->getValueType(0),
6754 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6755 AArch64::FCLAMP_VG2_2Z2Z_D}))
6756 SelectClamp(Node, 2,
Op);
6758 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6759 SelectClamp(Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
6761 case Intrinsic::aarch64_sve_sclamp_single_x4:
6763 Node->getValueType(0),
6764 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6765 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6766 SelectClamp(Node, 4,
Op);
6768 case Intrinsic::aarch64_sve_uclamp_single_x4:
6770 Node->getValueType(0),
6771 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6772 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6773 SelectClamp(Node, 4,
Op);
6775 case Intrinsic::aarch64_sve_fclamp_single_x4:
6777 Node->getValueType(0),
6778 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6779 AArch64::FCLAMP_VG4_4Z4Z_D}))
6780 SelectClamp(Node, 4,
Op);
6782 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6783 SelectClamp(Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
6785 case Intrinsic::aarch64_sve_add_single_x2:
6787 Node->getValueType(0),
6788 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6789 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6790 SelectDestructiveMultiIntrinsic(Node, 2,
false,
Op);
6792 case Intrinsic::aarch64_sve_add_single_x4:
6794 Node->getValueType(0),
6795 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6796 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6797 SelectDestructiveMultiIntrinsic(Node, 4,
false,
Op);
6799 case Intrinsic::aarch64_sve_zip_x2:
6801 Node->getValueType(0),
6802 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6803 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6804 SelectUnaryMultiIntrinsic(Node, 2,
false,
Op);
6806 case Intrinsic::aarch64_sve_zipq_x2:
6807 SelectUnaryMultiIntrinsic(Node, 2,
false,
6808 AArch64::ZIP_VG2_2ZZZ_Q);
6810 case Intrinsic::aarch64_sve_zip_x4:
6812 Node->getValueType(0),
6813 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6814 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6815 SelectUnaryMultiIntrinsic(Node, 4,
true,
Op);
6817 case Intrinsic::aarch64_sve_zipq_x4:
6818 SelectUnaryMultiIntrinsic(Node, 4,
true,
6819 AArch64::ZIP_VG4_4Z4Z_Q);
6821 case Intrinsic::aarch64_sve_uzp_x2:
6823 Node->getValueType(0),
6824 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6825 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6826 SelectUnaryMultiIntrinsic(Node, 2,
false,
Op);
6828 case Intrinsic::aarch64_sve_uzpq_x2:
6829 SelectUnaryMultiIntrinsic(Node, 2,
false,
6830 AArch64::UZP_VG2_2ZZZ_Q);
6832 case Intrinsic::aarch64_sve_uzp_x4:
6834 Node->getValueType(0),
6835 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6836 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6837 SelectUnaryMultiIntrinsic(Node, 4,
true,
Op);
6839 case Intrinsic::aarch64_sve_uzpq_x4:
6840 SelectUnaryMultiIntrinsic(Node, 4,
true,
6841 AArch64::UZP_VG4_4Z4Z_Q);
6843 case Intrinsic::aarch64_sve_sel_x2:
6845 Node->getValueType(0),
6846 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6847 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6848 SelectDestructiveMultiIntrinsic(Node, 2,
true,
Op,
true);
6850 case Intrinsic::aarch64_sve_sel_x4:
6852 Node->getValueType(0),
6853 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6854 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6855 SelectDestructiveMultiIntrinsic(Node, 4,
true,
Op,
true);
6857 case Intrinsic::aarch64_sve_frinta_x2:
6858 SelectFrintFromVT(Node, 2, AArch64::FRINTA_2Z2Z_S);
6860 case Intrinsic::aarch64_sve_frinta_x4:
6861 SelectFrintFromVT(Node, 4, AArch64::FRINTA_4Z4Z_S);
6863 case Intrinsic::aarch64_sve_frintm_x2:
6864 SelectFrintFromVT(Node, 2, AArch64::FRINTM_2Z2Z_S);
6866 case Intrinsic::aarch64_sve_frintm_x4:
6867 SelectFrintFromVT(Node, 4, AArch64::FRINTM_4Z4Z_S);
6869 case Intrinsic::aarch64_sve_frintn_x2:
6870 SelectFrintFromVT(Node, 2, AArch64::FRINTN_2Z2Z_S);
6872 case Intrinsic::aarch64_sve_frintn_x4:
6873 SelectFrintFromVT(Node, 4, AArch64::FRINTN_4Z4Z_S);
6875 case Intrinsic::aarch64_sve_frintp_x2:
6876 SelectFrintFromVT(Node, 2, AArch64::FRINTP_2Z2Z_S);
6878 case Intrinsic::aarch64_sve_frintp_x4:
6879 SelectFrintFromVT(Node, 4, AArch64::FRINTP_4Z4Z_S);
6881 case Intrinsic::aarch64_sve_sunpk_x2:
6883 Node->getValueType(0),
6884 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6885 AArch64::SUNPK_VG2_2ZZ_D}))
6886 SelectUnaryMultiIntrinsic(Node, 2,
false,
Op);
6888 case Intrinsic::aarch64_sve_uunpk_x2:
6890 Node->getValueType(0),
6891 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6892 AArch64::UUNPK_VG2_2ZZ_D}))
6893 SelectUnaryMultiIntrinsic(Node, 2,
false,
Op);
6895 case Intrinsic::aarch64_sve_sunpk_x4:
6897 Node->getValueType(0),
6898 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6899 AArch64::SUNPK_VG4_4Z2Z_D}))
6900 SelectUnaryMultiIntrinsic(Node, 4,
true,
Op);
6902 case Intrinsic::aarch64_sve_uunpk_x4:
6904 Node->getValueType(0),
6905 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6906 AArch64::UUNPK_VG4_4Z2Z_D}))
6907 SelectUnaryMultiIntrinsic(Node, 4,
true,
Op);
6909 case Intrinsic::aarch64_sve_pext_x2: {
6911 Node->getValueType(0),
6912 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6913 AArch64::PEXT_2PCI_D}))
6914 SelectPExtPair(Node,
Op);
6921 unsigned IntNo =
Node->getConstantOperandVal(1);
6922 if (
Node->getNumOperands() >= 3)
6923 VT =
Node->getOperand(2)->getValueType(0);
6927 case Intrinsic::aarch64_neon_st1x2: {
6928 if (VT == MVT::v8i8) {
6929 SelectStore(Node, 2, AArch64::ST1Twov8b);
6931 }
else if (VT == MVT::v16i8) {
6932 SelectStore(Node, 2, AArch64::ST1Twov16b);
6934 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6935 VT == MVT::v4bf16) {
6936 SelectStore(Node, 2, AArch64::ST1Twov4h);
6938 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6939 VT == MVT::v8bf16) {
6940 SelectStore(Node, 2, AArch64::ST1Twov8h);
6942 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6943 SelectStore(Node, 2, AArch64::ST1Twov2s);
6945 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6946 SelectStore(Node, 2, AArch64::ST1Twov4s);
6948 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6949 SelectStore(Node, 2, AArch64::ST1Twov2d);
6951 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6952 SelectStore(Node, 2, AArch64::ST1Twov1d);
6957 case Intrinsic::aarch64_neon_st1x3: {
6958 if (VT == MVT::v8i8) {
6959 SelectStore(Node, 3, AArch64::ST1Threev8b);
6961 }
else if (VT == MVT::v16i8) {
6962 SelectStore(Node, 3, AArch64::ST1Threev16b);
6964 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6965 VT == MVT::v4bf16) {
6966 SelectStore(Node, 3, AArch64::ST1Threev4h);
6968 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6969 VT == MVT::v8bf16) {
6970 SelectStore(Node, 3, AArch64::ST1Threev8h);
6972 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6973 SelectStore(Node, 3, AArch64::ST1Threev2s);
6975 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6976 SelectStore(Node, 3, AArch64::ST1Threev4s);
6978 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6979 SelectStore(Node, 3, AArch64::ST1Threev2d);
6981 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6982 SelectStore(Node, 3, AArch64::ST1Threev1d);
6987 case Intrinsic::aarch64_neon_st1x4: {
6988 if (VT == MVT::v8i8) {
6989 SelectStore(Node, 4, AArch64::ST1Fourv8b);
6991 }
else if (VT == MVT::v16i8) {
6992 SelectStore(Node, 4, AArch64::ST1Fourv16b);
6994 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6995 VT == MVT::v4bf16) {
6996 SelectStore(Node, 4, AArch64::ST1Fourv4h);
6998 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6999 VT == MVT::v8bf16) {
7000 SelectStore(Node, 4, AArch64::ST1Fourv8h);
7002 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7003 SelectStore(Node, 4, AArch64::ST1Fourv2s);
7005 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7006 SelectStore(Node, 4, AArch64::ST1Fourv4s);
7008 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7009 SelectStore(Node, 4, AArch64::ST1Fourv2d);
7011 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7012 SelectStore(Node, 4, AArch64::ST1Fourv1d);
7017 case Intrinsic::aarch64_neon_st2: {
7018 if (VT == MVT::v8i8) {
7019 SelectStore(Node, 2, AArch64::ST2Twov8b);
7021 }
else if (VT == MVT::v16i8) {
7022 SelectStore(Node, 2, AArch64::ST2Twov16b);
7024 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7025 VT == MVT::v4bf16) {
7026 SelectStore(Node, 2, AArch64::ST2Twov4h);
7028 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7029 VT == MVT::v8bf16) {
7030 SelectStore(Node, 2, AArch64::ST2Twov8h);
7032 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7033 SelectStore(Node, 2, AArch64::ST2Twov2s);
7035 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7036 SelectStore(Node, 2, AArch64::ST2Twov4s);
7038 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7039 SelectStore(Node, 2, AArch64::ST2Twov2d);
7041 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7042 SelectStore(Node, 2, AArch64::ST1Twov1d);
7047 case Intrinsic::aarch64_neon_st3: {
7048 if (VT == MVT::v8i8) {
7049 SelectStore(Node, 3, AArch64::ST3Threev8b);
7051 }
else if (VT == MVT::v16i8) {
7052 SelectStore(Node, 3, AArch64::ST3Threev16b);
7054 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7055 VT == MVT::v4bf16) {
7056 SelectStore(Node, 3, AArch64::ST3Threev4h);
7058 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7059 VT == MVT::v8bf16) {
7060 SelectStore(Node, 3, AArch64::ST3Threev8h);
7062 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7063 SelectStore(Node, 3, AArch64::ST3Threev2s);
7065 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7066 SelectStore(Node, 3, AArch64::ST3Threev4s);
7068 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7069 SelectStore(Node, 3, AArch64::ST3Threev2d);
7071 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7072 SelectStore(Node, 3, AArch64::ST1Threev1d);
7077 case Intrinsic::aarch64_neon_st4: {
7078 if (VT == MVT::v8i8) {
7079 SelectStore(Node, 4, AArch64::ST4Fourv8b);
7081 }
else if (VT == MVT::v16i8) {
7082 SelectStore(Node, 4, AArch64::ST4Fourv16b);
7084 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7085 VT == MVT::v4bf16) {
7086 SelectStore(Node, 4, AArch64::ST4Fourv4h);
7088 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7089 VT == MVT::v8bf16) {
7090 SelectStore(Node, 4, AArch64::ST4Fourv8h);
7092 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7093 SelectStore(Node, 4, AArch64::ST4Fourv2s);
7095 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7096 SelectStore(Node, 4, AArch64::ST4Fourv4s);
7098 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7099 SelectStore(Node, 4, AArch64::ST4Fourv2d);
7101 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7102 SelectStore(Node, 4, AArch64::ST1Fourv1d);
7107 case Intrinsic::aarch64_neon_st2lane: {
7108 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7109 SelectStoreLane(Node, 2, AArch64::ST2i8);
7111 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7112 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7113 SelectStoreLane(Node, 2, AArch64::ST2i16);
7115 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7117 SelectStoreLane(Node, 2, AArch64::ST2i32);
7119 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7121 SelectStoreLane(Node, 2, AArch64::ST2i64);
7126 case Intrinsic::aarch64_neon_st3lane: {
7127 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7128 SelectStoreLane(Node, 3, AArch64::ST3i8);
7130 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7131 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7132 SelectStoreLane(Node, 3, AArch64::ST3i16);
7134 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7136 SelectStoreLane(Node, 3, AArch64::ST3i32);
7138 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7140 SelectStoreLane(Node, 3, AArch64::ST3i64);
7145 case Intrinsic::aarch64_neon_st4lane: {
7146 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7147 SelectStoreLane(Node, 4, AArch64::ST4i8);
7149 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7150 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7151 SelectStoreLane(Node, 4, AArch64::ST4i16);
7153 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7155 SelectStoreLane(Node, 4, AArch64::ST4i32);
7157 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7159 SelectStoreLane(Node, 4, AArch64::ST4i64);
7164 case Intrinsic::aarch64_sve_st2q: {
7165 SelectPredicatedStore(Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
7168 case Intrinsic::aarch64_sve_st3q: {
7169 SelectPredicatedStore(Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
7172 case Intrinsic::aarch64_sve_st4q: {
7173 SelectPredicatedStore(Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
7176 case Intrinsic::aarch64_sve_st2: {
7177 if (VT == MVT::nxv16i8) {
7178 SelectPredicatedStore(Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
7180 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7181 VT == MVT::nxv8bf16) {
7182 SelectPredicatedStore(Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
7184 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7185 SelectPredicatedStore(Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
7187 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7188 SelectPredicatedStore(Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
7193 case Intrinsic::aarch64_sve_st3: {
7194 if (VT == MVT::nxv16i8) {
7195 SelectPredicatedStore(Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
7197 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7198 VT == MVT::nxv8bf16) {
7199 SelectPredicatedStore(Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
7201 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7202 SelectPredicatedStore(Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
7204 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7205 SelectPredicatedStore(Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
7210 case Intrinsic::aarch64_sve_st4: {
7211 if (VT == MVT::nxv16i8) {
7212 SelectPredicatedStore(Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
7214 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7215 VT == MVT::nxv8bf16) {
7216 SelectPredicatedStore(Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
7218 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7219 SelectPredicatedStore(Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
7221 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7222 SelectPredicatedStore(Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
7230 case AArch64ISD::LD2post: {
7231 if (VT == MVT::v8i8) {
7232 SelectPostLoad(Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
7234 }
else if (VT == MVT::v16i8) {
7235 SelectPostLoad(Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
7237 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7238 SelectPostLoad(Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
7240 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7241 SelectPostLoad(Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
7243 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7244 SelectPostLoad(Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
7246 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7247 SelectPostLoad(Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
7249 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7250 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
7252 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7253 SelectPostLoad(Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
7258 case AArch64ISD::LD3post: {
7259 if (VT == MVT::v8i8) {
7260 SelectPostLoad(Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
7262 }
else if (VT == MVT::v16i8) {
7263 SelectPostLoad(Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
7265 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7266 SelectPostLoad(Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
7268 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7269 SelectPostLoad(Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
7271 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7272 SelectPostLoad(Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
7274 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7275 SelectPostLoad(Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
7277 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7278 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
7280 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7281 SelectPostLoad(Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
7286 case AArch64ISD::LD4post: {
7287 if (VT == MVT::v8i8) {
7288 SelectPostLoad(Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
7290 }
else if (VT == MVT::v16i8) {
7291 SelectPostLoad(Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
7293 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7294 SelectPostLoad(Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
7296 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7297 SelectPostLoad(Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
7299 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7300 SelectPostLoad(Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
7302 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7303 SelectPostLoad(Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
7305 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7306 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
7308 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7309 SelectPostLoad(Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
7314 case AArch64ISD::LD1x2post: {
7315 if (VT == MVT::v8i8) {
7316 SelectPostLoad(Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
7318 }
else if (VT == MVT::v16i8) {
7319 SelectPostLoad(Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
7321 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7322 SelectPostLoad(Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
7324 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7325 SelectPostLoad(Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
7327 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7328 SelectPostLoad(Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
7330 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7331 SelectPostLoad(Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
7333 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7334 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
7336 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7337 SelectPostLoad(Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
7342 case AArch64ISD::LD1x3post: {
7343 if (VT == MVT::v8i8) {
7344 SelectPostLoad(Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
7346 }
else if (VT == MVT::v16i8) {
7347 SelectPostLoad(Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
7349 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7350 SelectPostLoad(Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
7352 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7353 SelectPostLoad(Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
7355 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7356 SelectPostLoad(Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
7358 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7359 SelectPostLoad(Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
7361 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7362 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
7364 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7365 SelectPostLoad(Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
7370 case AArch64ISD::LD1x4post: {
7371 if (VT == MVT::v8i8) {
7372 SelectPostLoad(Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
7374 }
else if (VT == MVT::v16i8) {
7375 SelectPostLoad(Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
7377 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7378 SelectPostLoad(Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
7380 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7381 SelectPostLoad(Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
7383 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7384 SelectPostLoad(Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
7386 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7387 SelectPostLoad(Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
7389 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7390 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
7392 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7393 SelectPostLoad(Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
7398 case AArch64ISD::LD1DUPpost: {
7399 if (VT == MVT::v8i8) {
7400 SelectPostLoad(Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
7402 }
else if (VT == MVT::v16i8) {
7403 SelectPostLoad(Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
7405 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7406 SelectPostLoad(Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
7408 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7409 SelectPostLoad(Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
7411 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7412 SelectPostLoad(Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
7414 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7415 SelectPostLoad(Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
7417 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7418 SelectPostLoad(Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
7420 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7421 SelectPostLoad(Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
7426 case AArch64ISD::LD2DUPpost: {
7427 if (VT == MVT::v8i8) {
7428 SelectPostLoad(Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
7430 }
else if (VT == MVT::v16i8) {
7431 SelectPostLoad(Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
7433 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7434 SelectPostLoad(Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
7436 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7437 SelectPostLoad(Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
7439 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7440 SelectPostLoad(Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
7442 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7443 SelectPostLoad(Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
7445 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7446 SelectPostLoad(Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
7448 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7449 SelectPostLoad(Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
7454 case AArch64ISD::LD3DUPpost: {
7455 if (VT == MVT::v8i8) {
7456 SelectPostLoad(Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
7458 }
else if (VT == MVT::v16i8) {
7459 SelectPostLoad(Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
7461 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7462 SelectPostLoad(Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
7464 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7465 SelectPostLoad(Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
7467 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7468 SelectPostLoad(Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
7470 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7471 SelectPostLoad(Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
7473 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7474 SelectPostLoad(Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
7476 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7477 SelectPostLoad(Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
7482 case AArch64ISD::LD4DUPpost: {
7483 if (VT == MVT::v8i8) {
7484 SelectPostLoad(Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
7486 }
else if (VT == MVT::v16i8) {
7487 SelectPostLoad(Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
7489 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7490 SelectPostLoad(Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
7492 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7493 SelectPostLoad(Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
7495 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7496 SelectPostLoad(Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
7498 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7499 SelectPostLoad(Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
7501 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7502 SelectPostLoad(Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
7504 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7505 SelectPostLoad(Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
7510 case AArch64ISD::LD1LANEpost: {
7511 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7512 SelectPostLoadLane(Node, 1, AArch64::LD1i8_POST);
7514 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7515 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7516 SelectPostLoadLane(Node, 1, AArch64::LD1i16_POST);
7518 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7520 SelectPostLoadLane(Node, 1, AArch64::LD1i32_POST);
7522 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7524 SelectPostLoadLane(Node, 1, AArch64::LD1i64_POST);
7529 case AArch64ISD::LD2LANEpost: {
7530 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7531 SelectPostLoadLane(Node, 2, AArch64::LD2i8_POST);
7533 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7534 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7535 SelectPostLoadLane(Node, 2, AArch64::LD2i16_POST);
7537 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7539 SelectPostLoadLane(Node, 2, AArch64::LD2i32_POST);
7541 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7543 SelectPostLoadLane(Node, 2, AArch64::LD2i64_POST);
7548 case AArch64ISD::LD3LANEpost: {
7549 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7550 SelectPostLoadLane(Node, 3, AArch64::LD3i8_POST);
7552 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7553 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7554 SelectPostLoadLane(Node, 3, AArch64::LD3i16_POST);
7556 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7558 SelectPostLoadLane(Node, 3, AArch64::LD3i32_POST);
7560 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7562 SelectPostLoadLane(Node, 3, AArch64::LD3i64_POST);
7567 case AArch64ISD::LD4LANEpost: {
7568 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7569 SelectPostLoadLane(Node, 4, AArch64::LD4i8_POST);
7571 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7572 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7573 SelectPostLoadLane(Node, 4, AArch64::LD4i16_POST);
7575 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7577 SelectPostLoadLane(Node, 4, AArch64::LD4i32_POST);
7579 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7581 SelectPostLoadLane(Node, 4, AArch64::LD4i64_POST);
7586 case AArch64ISD::ST2post: {
7587 VT =
Node->getOperand(1).getValueType();
7588 if (VT == MVT::v8i8) {
7589 SelectPostStore(Node, 2, AArch64::ST2Twov8b_POST);
7591 }
else if (VT == MVT::v16i8) {
7592 SelectPostStore(Node, 2, AArch64::ST2Twov16b_POST);
7594 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7595 SelectPostStore(Node, 2, AArch64::ST2Twov4h_POST);
7597 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7598 SelectPostStore(Node, 2, AArch64::ST2Twov8h_POST);
7600 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7601 SelectPostStore(Node, 2, AArch64::ST2Twov2s_POST);
7603 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7604 SelectPostStore(Node, 2, AArch64::ST2Twov4s_POST);
7606 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7607 SelectPostStore(Node, 2, AArch64::ST2Twov2d_POST);
7609 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7610 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
7615 case AArch64ISD::ST3post: {
7616 VT =
Node->getOperand(1).getValueType();
7617 if (VT == MVT::v8i8) {
7618 SelectPostStore(Node, 3, AArch64::ST3Threev8b_POST);
7620 }
else if (VT == MVT::v16i8) {
7621 SelectPostStore(Node, 3, AArch64::ST3Threev16b_POST);
7623 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7624 SelectPostStore(Node, 3, AArch64::ST3Threev4h_POST);
7626 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7627 SelectPostStore(Node, 3, AArch64::ST3Threev8h_POST);
7629 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7630 SelectPostStore(Node, 3, AArch64::ST3Threev2s_POST);
7632 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7633 SelectPostStore(Node, 3, AArch64::ST3Threev4s_POST);
7635 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7636 SelectPostStore(Node, 3, AArch64::ST3Threev2d_POST);
7638 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7639 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
7644 case AArch64ISD::ST4post: {
7645 VT =
Node->getOperand(1).getValueType();
7646 if (VT == MVT::v8i8) {
7647 SelectPostStore(Node, 4, AArch64::ST4Fourv8b_POST);
7649 }
else if (VT == MVT::v16i8) {
7650 SelectPostStore(Node, 4, AArch64::ST4Fourv16b_POST);
7652 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7653 SelectPostStore(Node, 4, AArch64::ST4Fourv4h_POST);
7655 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7656 SelectPostStore(Node, 4, AArch64::ST4Fourv8h_POST);
7658 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7659 SelectPostStore(Node, 4, AArch64::ST4Fourv2s_POST);
7661 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7662 SelectPostStore(Node, 4, AArch64::ST4Fourv4s_POST);
7664 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7665 SelectPostStore(Node, 4, AArch64::ST4Fourv2d_POST);
7667 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7668 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
7673 case AArch64ISD::ST1x2post: {
7674 VT =
Node->getOperand(1).getValueType();
7675 if (VT == MVT::v8i8) {
7676 SelectPostStore(Node, 2, AArch64::ST1Twov8b_POST);
7678 }
else if (VT == MVT::v16i8) {
7679 SelectPostStore(Node, 2, AArch64::ST1Twov16b_POST);
7681 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7682 SelectPostStore(Node, 2, AArch64::ST1Twov4h_POST);
7684 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7685 SelectPostStore(Node, 2, AArch64::ST1Twov8h_POST);
7687 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7688 SelectPostStore(Node, 2, AArch64::ST1Twov2s_POST);
7690 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7691 SelectPostStore(Node, 2, AArch64::ST1Twov4s_POST);
7693 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7694 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
7696 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7697 SelectPostStore(Node, 2, AArch64::ST1Twov2d_POST);
7702 case AArch64ISD::ST1x3post: {
7703 VT =
Node->getOperand(1).getValueType();
7704 if (VT == MVT::v8i8) {
7705 SelectPostStore(Node, 3, AArch64::ST1Threev8b_POST);
7707 }
else if (VT == MVT::v16i8) {
7708 SelectPostStore(Node, 3, AArch64::ST1Threev16b_POST);
7710 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7711 SelectPostStore(Node, 3, AArch64::ST1Threev4h_POST);
7713 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7714 SelectPostStore(Node, 3, AArch64::ST1Threev8h_POST);
7716 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7717 SelectPostStore(Node, 3, AArch64::ST1Threev2s_POST);
7719 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7720 SelectPostStore(Node, 3, AArch64::ST1Threev4s_POST);
7722 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7723 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
7725 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7726 SelectPostStore(Node, 3, AArch64::ST1Threev2d_POST);
7731 case AArch64ISD::ST1x4post: {
7732 VT =
Node->getOperand(1).getValueType();
7733 if (VT == MVT::v8i8) {
7734 SelectPostStore(Node, 4, AArch64::ST1Fourv8b_POST);
7736 }
else if (VT == MVT::v16i8) {
7737 SelectPostStore(Node, 4, AArch64::ST1Fourv16b_POST);
7739 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7740 SelectPostStore(Node, 4, AArch64::ST1Fourv4h_POST);
7742 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7743 SelectPostStore(Node, 4, AArch64::ST1Fourv8h_POST);
7745 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7746 SelectPostStore(Node, 4, AArch64::ST1Fourv2s_POST);
7748 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7749 SelectPostStore(Node, 4, AArch64::ST1Fourv4s_POST);
7751 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7752 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
7754 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7755 SelectPostStore(Node, 4, AArch64::ST1Fourv2d_POST);
7760 case AArch64ISD::ST2LANEpost: {
7761 VT =
Node->getOperand(1).getValueType();
7762 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7763 SelectPostStoreLane(Node, 2, AArch64::ST2i8_POST);
7765 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7766 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7767 SelectPostStoreLane(Node, 2, AArch64::ST2i16_POST);
7769 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7771 SelectPostStoreLane(Node, 2, AArch64::ST2i32_POST);
7773 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7775 SelectPostStoreLane(Node, 2, AArch64::ST2i64_POST);
7780 case AArch64ISD::ST3LANEpost: {
7781 VT =
Node->getOperand(1).getValueType();
7782 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7783 SelectPostStoreLane(Node, 3, AArch64::ST3i8_POST);
7785 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7786 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7787 SelectPostStoreLane(Node, 3, AArch64::ST3i16_POST);
7789 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7791 SelectPostStoreLane(Node, 3, AArch64::ST3i32_POST);
7793 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7795 SelectPostStoreLane(Node, 3, AArch64::ST3i64_POST);
7800 case AArch64ISD::ST4LANEpost: {
7801 VT =
Node->getOperand(1).getValueType();
7802 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7803 SelectPostStoreLane(Node, 4, AArch64::ST4i8_POST);
7805 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7806 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7807 SelectPostStoreLane(Node, 4, AArch64::ST4i16_POST);
7809 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7811 SelectPostStoreLane(Node, 4, AArch64::ST4i32_POST);
7813 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7815 SelectPostStoreLane(Node, 4, AArch64::ST4i64_POST);
7830 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7842 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
7846 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7847 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7869 return MemIntr->getMemoryVT();
7876 DataVT =
Load->getValueType(0);
7878 DataVT =
Load->getValueType(0);
7880 DataVT =
Store->getValue().getValueType();
7882 DataVT =
Store->getValue().getValueType();
7889 const unsigned Opcode = Root->
getOpcode();
7893 case AArch64ISD::LD1_MERGE_ZERO:
7894 case AArch64ISD::LD1S_MERGE_ZERO:
7895 case AArch64ISD::LDNF1_MERGE_ZERO:
7896 case AArch64ISD::LDNF1S_MERGE_ZERO:
7898 case AArch64ISD::ST1_PRED:
7910 case Intrinsic::aarch64_sme_ldr:
7911 case Intrinsic::aarch64_sme_str:
7912 return MVT::nxv16i8;
7913 case Intrinsic::aarch64_sve_prf:
7918 case Intrinsic::aarch64_sve_ld2_sret:
7919 case Intrinsic::aarch64_sve_ld2q_sret:
7922 case Intrinsic::aarch64_sve_st2q:
7925 case Intrinsic::aarch64_sve_ld3_sret:
7926 case Intrinsic::aarch64_sve_ld3q_sret:
7929 case Intrinsic::aarch64_sve_st3q:
7932 case Intrinsic::aarch64_sve_ld4_sret:
7933 case Intrinsic::aarch64_sve_ld4q_sret:
7936 case Intrinsic::aarch64_sve_st4q:
7939 case Intrinsic::aarch64_sve_ld1_pn_x2:
7940 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7943 case Intrinsic::aarch64_sve_ld1_pn_x4:
7944 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7947 case Intrinsic::aarch64_sve_st1_pn_x2:
7948 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7951 case Intrinsic::aarch64_sve_st1_pn_x4:
7952 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7955 case Intrinsic::aarch64_sve_ld1udq:
7956 case Intrinsic::aarch64_sve_st1dq:
7957 return EVT(MVT::nxv1i64);
7958 case Intrinsic::aarch64_sve_ld1uwq:
7959 case Intrinsic::aarch64_sve_st1wq:
7960 return EVT(MVT::nxv1i32);
7967template <
int64_t Min,
int64_t Max>
7968bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root,
SDValue N,
7972 const DataLayout &
DL = CurDAG->getDataLayout();
7973 const MachineFrameInfo &MFI = MF->getFrameInfo();
7981 OffImm = CurDAG->getTargetConstant(0, SDLoc(
N), MVT::i64);
7995 int64_t MulImm = std::numeric_limits<int64_t>::max();
7999 int64_t ByteOffset =
C->getSExtValue();
8000 const auto KnownVScale =
8003 if (!KnownVScale || ByteOffset % KnownVScale != 0)
8006 MulImm = ByteOffset / KnownVScale;
8013 if ((MulImm % MemWidthBytes) != 0)
8016 int64_t
Offset = MulImm / MemWidthBytes;
8020 Base =
N.getOperand(0);
8029 OffImm = CurDAG->getTargetConstant(
Offset, SDLoc(
N), MVT::i64);
8035bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
8054 int64_t ImmOff =
C->getSExtValue();
8055 unsigned Size = 1 << Scale;
8064 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
8066 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64,
Ops);
8077 if (
C->getZExtValue() == Scale) {
8086bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
8087 const AArch64TargetLowering *
TLI =
8088 static_cast<const AArch64TargetLowering *
>(getTargetLowering());
8090 return TLI->isAllActivePredicate(*CurDAG,
N);
8093bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
8097bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
8102 int64_t ImmOff =
C->getSExtValue();
8103 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8104 return CurDAG->getTargetConstant(ImmOff / Scale, SDLoc(
N), MVT::i64);
8109 if (
SDValue C = MatchConstantOffset(
N)) {
8116 if (CurDAG->isBaseWithConstantOffset(
N)) {
8117 if (
SDValue C = MatchConstantOffset(
N.getOperand(1))) {
8118 Base =
N.getOperand(0);
8126 Offset = CurDAG->getTargetConstant(0, SDLoc(
N), MVT::i64);
8130bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *
P,
SDValue N,
8168 if (CN->getAPIntValue().uge(LowerBound) &&
8169 CN->getAPIntValue().ult(UpperBound)) {
8171 Imm = CurDAG->getTargetConstant(CN->getZExtValue(),
DL,
N.getValueType());
8179template <
bool MatchCBB>
8186 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8188 Reg =
N.getOperand(0);
8190 SDLoc(
N), MVT::i32);
8198 Reg =
N.getOperand(0);
8218bool AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode *
N) {
8219 EVT VT =
N->getValueType(0);
8230 if (
Cmp.getOpcode() != AArch64ISD::FCMP)
8242 auto getOpc = [](EVT VT,
bool isMax) ->
unsigned {
8244 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8245 else if (VT == MVT::f32)
8246 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8247 else if (VT == MVT::f64)
8248 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8256 if (TVal == CmpLHS && FVal == CmpRHS)
8261 if (TVal == CmpLHS && FVal == CmpRHS)
8270 unsigned Opc = getOpc(VT, isMax);
8276 if (!CFP || CFP->getValueAPF().isNaN())
8281 if (CFP->isZero() && !
N->getFlags().hasNoSignedZeros())
8287 if (!CurDAG->isKnownNeverSNaN(CmpLHS))
8290 CurDAG->SelectNodeTo(
N,
Opc, VT, CmpLHS, CmpRHS);
8294void AArch64DAGToDAGISel::PreprocessISelDAG() {
8295 bool MadeChange =
false;
8301 switch (
N.getOpcode()) {
8303 EVT ScalarTy =
N.getValueType(0).getVectorElementType();
8304 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8305 ScalarTy ==
N.getOperand(0).getValueType())
8310 case AArch64ISD::VSHL: {
8313 EVT VT =
N.getValueType(0);
8320 if (
B.getOpcode() ==
A.getOpcode())
8333 LLVM_DEBUG(
dbgs() <<
"AArch64 DAG preprocessing replacing:\nOld: ");
8339 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(&
N, 0), Result);
8345 CurDAG->RemoveDeadNodes();
static std::optional< APInt > GetNEONSplatValue(SDValue N, const AArch64Subtarget *Subtarget)
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool allUsesAreShiftAmounts(SDValue N)
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N)
addBitcastHints - This method adds bitcast hints to the operands of a node to help instruction select...
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT)
Returns a copy from WZR or XZR.
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static bool isMemOpOrPrefetch(SDNode *N)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static std::optional< APInt > DecodeNEONSplat(SDValue N, const AArch64Subtarget *Subtarget)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec)
Builds an integer vector type large enough to hold NumVec instances of VecVT.
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG)
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 regis...
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Promote Memory to Register
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
OptimizedStructLayoutField Field
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Contains matchers for matching SelectionDAG nodes and values.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
const AArch64InstrInfo * getInstrInfo() const override
const AArch64TargetLowering * getTargetLowering() const override
bool isLittleEndian() const
bool isStreaming() const
Returns true if the function has a streaming body.
bool isX16X17Safer() const
Returns whether the operating system makes it safer to store sensitive values in x16 and x17 as oppos...
unsigned getSVEVectorSizeInBits() const
Register matchRegisterName(StringRef RegName) const
static const fltSemantics & IEEEsingle()
static const fltSemantics & IEEEdouble()
static const fltSemantics & IEEEhalf()
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
const TargetRegisterClass * getInlineAsmMemoryOperandRegClass(InlineAsm::ConstraintCode C) const override
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getID() const
getID() - Return the register class ID number.
const MDOperand & getOperand(unsigned I) const
unsigned getNumOperands() const
Return number of MDNode operands.
bool equalsStr(StringRef Str) const
uint64_t getScalarSizeInBits() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool hasScalableStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
const MDNode * getMemCacheHint() const
Return the cache hint metadata for the memory reference.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
static constexpr unsigned MaxRecursionDepth
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint64_t decodeAdvSIMDModImmType12(uint8_t Imm)
constexpr bool isLegalArithImmed(const uint64_t C)
isLegalArithImmed -
static uint64_t decodeAdvSIMDModImmType11(uint8_t Imm)
unsigned getExtendEncoding(AArch64_AM::ShiftExtendType ET)
Mapping from extend bits to required operation: shifter: 000 ==> uxtb 001 ==> uxth 010 ==> uxtw 011 =...
static uint64_t decodeAdvSIMDModImmType10(uint8_t Imm)
static bool isSVELogicalImm(unsigned SizeInBits, uint64_t ImmVal, uint64_t &Encoding)
constexpr unsigned getArithImmedShift(const uint64_t C)
getArithImmedShift - assumes C is a legal immediate for arithmetic instructions and
static bool isSVECpyDupImm(int SizeInBits, int64_t Val, int32_t &Imm, int32_t &Shift)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type)
isSignExtendShiftType - Returns true if Type is sign extending.
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static constexpr unsigned SVEBitsPerBlock
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
@ POISON
POISON - A poison node.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
BinaryOpc_match< LHS, RHS, true > m_Mul(const LHS &L, const RHS &R)
Or< Preds... > m_AnyOf(const Preds &...preds)
auto m_SExt(const Opnd &Op)
bool sd_match(SDValue N, Pattern &&P)
UnaryOpc_match< Opnd > m_ZExt(const Opnd &Op)
Value_match m_Value()
Match any valid SDValue.
NUses_match< 1, Value_match > m_OneUse()
Not(const Pred &P) -> Not< Pred >
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< NodeBase * > Node
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned CheckFixedPointOperandConstant(APFloat &FVal, unsigned RegWidth, bool isReciprocal)
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
AArch64MemoryHint toAArch64MemoryHint(Int I)
MCRegisterClass TargetRegisterClass
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool isScalableVectorOf(EVT EltVT) const
Return true if this is a scalable vector with matching element type.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.