22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
35#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
36#pragma inline_depth(0)
52 AArch64DAGToDAGISel() =
delete;
69 std::vector<SDValue> &OutOps)
override;
71 template <
signed Low,
signed High,
signed Scale>
79 return SelectShiftedRegister(
N,
false, Reg, Shift);
82 return SelectShiftedRegister(
N,
true, Reg, Shift);
85 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
88 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
91 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
94 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
97 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
100 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
103 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
106 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
109 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
112 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
115 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
118 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
121 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
124 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
127 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
130 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
133 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
135 template <
unsigned Size,
unsigned Max>
139 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
141 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
142 int64_t
C = CI->getSExtValue();
150 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
157 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
163 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
168 N =
N->getOperand(0);
170 !isa<ConstantSDNode>(
N->getOperand(1)))
172 EVT VT =
N->getValueType(0);
173 EVT LVT =
N->getOperand(0).getValueType();
174 unsigned Index =
N->getConstantOperandVal(1);
178 Res =
N->getOperand(0);
186 EVT VT =
Op.getValueType();
187 unsigned ShtAmt =
N->getConstantOperandVal(1);
194 Op.getOperand(1).getConstantOperandVal(0)
195 <<
Op.getOperand(1).getConstantOperandVal(1));
197 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
199 Op.getOperand(1).getConstantOperandVal(0));
203 if (Imm != 1ULL << (ShtAmt - 1))
206 Res1 =
Op.getOperand(0);
207 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
211 bool SelectDupZeroOrUndef(
SDValue N) {
212 switch(
N->getOpcode()) {
217 auto Opnd0 =
N->getOperand(0);
231 bool SelectAny(
SDValue) {
return true; }
234 switch(
N->getOpcode()) {
237 auto Opnd0 =
N->getOperand(0);
249 template<MVT::SimpleValueType VT>
251 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
254 template <MVT::SimpleValueType VT,
bool Negate>
256 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
259 template <MVT::SimpleValueType VT>
261 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
264 template <MVT::SimpleValueType VT,
bool Invert = false>
266 return SelectSVELogicalImm(
N, VT, Imm, Invert);
269 template <MVT::SimpleValueType VT>
271 return SelectSVEArithImm(
N, VT, Imm);
274 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
276 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
283 EVT EltVT =
N->getValueType(0).getVectorElementType();
284 return SelectSVEShiftImm(
N->getOperand(0), 1,
290 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
292 if (!isa<ConstantSDNode>(
N))
295 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
297 MulImm = 1LL << MulImm;
299 if ((MulImm % std::abs(Scale)) != 0)
303 if ((MulImm >= Min) && (MulImm <= Max)) {
304 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
311 template <
signed Max,
signed Scale>
313 if (!isa<ConstantSDNode>(
N))
316 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
318 if (MulImm >= 0 && MulImm <= Max) {
320 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
327 template <
unsigned BaseReg,
unsigned Max>
329 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
335 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
358 const unsigned SubRegs[]);
360 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
362 bool tryIndexedLoad(
SDNode *
N);
364 void SelectPtrauthAuth(
SDNode *
N);
365 void SelectPtrauthResign(
SDNode *
N);
367 bool trySelectStackSlotTagP(
SDNode *
N);
370 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
372 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
374 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
375 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
376 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
377 unsigned Opc_rr,
unsigned Opc_ri,
378 bool IsIntr =
false);
379 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
380 unsigned Scale,
unsigned Opc_ri,
382 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
383 bool IsZmMulti,
unsigned Opcode,
384 bool HasPred =
false);
385 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
386 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
387 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
388 void SelectCVTIntrinsicFP8(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
389 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
390 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
391 bool IsTupleInput,
unsigned Opc);
392 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
394 template <
unsigned MaxIdx,
unsigned Scale>
395 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
397 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
398 unsigned Op,
unsigned MaxIdx,
unsigned Scale,
399 unsigned BaseReg = 0);
402 template <
int64_t Min,
int64_t Max>
406 template <
unsigned Scale>
408 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
411 void SelectMultiVectorLutiLane(
SDNode *
Node,
unsigned NumOutVecs,
414 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc);
416 template <
unsigned MaxIdx,
unsigned Scale>
421 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
422 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
423 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
424 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
425 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
426 unsigned Opc_rr,
unsigned Opc_ri);
427 std::tuple<unsigned, SDValue, SDValue>
428 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
432 bool tryBitfieldExtractOp(
SDNode *
N);
433 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
434 bool tryBitfieldInsertOp(
SDNode *
N);
435 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
436 bool tryShiftAmountMod(
SDNode *
N);
438 bool tryReadRegister(
SDNode *
N);
439 bool tryWriteRegister(
SDNode *
N);
441 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
442 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
447#include "AArch64GenDAGISel.inc"
455 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
457 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
470 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
471 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
472 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
475 template<
unsigned RegW
idth>
477 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
480 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
482 template<
unsigned RegW
idth>
484 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
490 bool SelectCMP_SWAP(
SDNode *
N);
500 bool AllowSaturation,
SDValue &Imm);
508 bool SelectAllActivePredicate(
SDValue N);
518 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
522char AArch64DAGToDAGISelLegacy::ID = 0;
530 Imm =
C->getZExtValue();
547 return N->getOpcode() == Opc &&
558 return Imm == ImmExpected;
562bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
564 std::vector<SDValue> &OutOps) {
565 switch(ConstraintID) {
568 case InlineAsm::ConstraintCode::m:
569 case InlineAsm::ConstraintCode::o:
570 case InlineAsm::ConstraintCode::Q:
576 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
578 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
579 dl,
Op.getValueType(),
581 OutOps.push_back(NewOp);
597 if (!isa<ConstantSDNode>(
N.getNode()))
600 uint64_t Immed =
N.getNode()->getAsZExtVal();
603 if (Immed >> 12 == 0) {
605 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
613 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
614 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
627 if (!isa<ConstantSDNode>(
N.getNode()))
631 uint64_t Immed =
N.getNode()->getAsZExtVal();
639 if (
N.getValueType() == MVT::i32)
642 Immed = ~Immed + 1ULL;
643 if (Immed & 0xFFFFFFFFFF000000ULL)
646 Immed &= 0xFFFFFFULL;
647 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
654 switch (
N.getOpcode()) {
677 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
680 unsigned ShiftVal = CSD->getZExtValue();
698bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
701 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
706 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
728bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
730 EVT VT =
N.getValueType();
731 if (VT != MVT::i32 && VT != MVT::i64)
734 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
740 unsigned LHSOpcode =
LHS->getOpcode();
754 unsigned LowZBits, MaskLen;
758 unsigned BitWidth =
N.getValueSizeInBits();
765 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
768 NewShiftC = LowZBits - ShiftAmtC;
769 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
775 NewShiftC = LowZBits + ShiftAmtC;
788 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
790 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
794 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
796 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
797 NewShiftAmt, BitWidthMinus1),
800 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
812 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
814 SrcVT =
N.getOperand(0).getValueType();
816 if (!IsLoadStore && SrcVT == MVT::i8)
818 else if (!IsLoadStore && SrcVT == MVT::i16)
820 else if (SrcVT == MVT::i32)
822 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
827 EVT SrcVT =
N.getOperand(0).getValueType();
828 if (!IsLoadStore && SrcVT == MVT::i8)
830 else if (!IsLoadStore && SrcVT == MVT::i16)
832 else if (SrcVT == MVT::i32)
834 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
862bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
865 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
870 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
871 V.getConstantOperandVal(1) <= 4 &&
884bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
886 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
896 unsigned BitSize =
N.getValueSizeInBits();
897 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
900 Reg =
N.getOperand(0);
901 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
902 return isWorthFoldingALU(
N,
true);
913 if (
N.getValueType() == MVT::i32)
921template<
signed Low,
signed High,
signed Scale>
923 if (!isa<ConstantSDNode>(
N))
926 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
927 if ((MulImm % std::abs(Scale)) == 0) {
928 int64_t RDVLImm = MulImm / Scale;
929 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
930 Imm = CurDAG->getSignedTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
940bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
942 unsigned ShiftVal = 0;
957 Reg =
N.getOperand(0).getOperand(0);
963 Reg =
N.getOperand(0);
968 unsigned Opc =
N.getOpcode();
969 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
986 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
988 return isWorthFoldingALU(
N);
993bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
995 unsigned ShiftVal = 0;
1009 Reg =
N.getOperand(0);
1010 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1012 return isWorthFoldingALU(
N);
1021 for (
auto *
User :
N->users()) {
1048bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1049 unsigned BW,
unsigned Size,
1056 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1058 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1064 if (CurDAG->isBaseWithConstantOffset(
N)) {
1065 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1067 int64_t RHSC =
RHS->getSExtValue();
1069 int64_t
Range = 0x1LL << (BW - 1);
1071 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1072 RHSC < (
Range << Scale)) {
1073 Base =
N.getOperand(0);
1075 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1078 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1087 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1088 Base =
N.getOperand(0);
1090 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1093 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1104 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1111bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1117 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1119 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1125 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1126 Base =
N.getOperand(0);
1127 OffImm =
N.getOperand(1);
1136 if (CurDAG->isBaseWithConstantOffset(
N)) {
1137 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1138 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1141 Base =
N.getOperand(0);
1143 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1146 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1154 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1162 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1171bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1174 if (!CurDAG->isBaseWithConstantOffset(
N))
1176 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1177 int64_t RHSC =
RHS->getSExtValue();
1178 if (RHSC >= -256 && RHSC < 256) {
1179 Base =
N.getOperand(0);
1181 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1183 Base = CurDAG->getTargetFrameIndex(
1186 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1196 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1203bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1223 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1229 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1232 return isWorthFoldingAddr(
N,
Size);
1235bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1247 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1260 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1263 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1264 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1266 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1271 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1272 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1274 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1279 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1283 if (IsExtendedRegisterWorthFolding &&
1290 if (isWorthFoldingAddr(LHS,
Size))
1295 if (IsExtendedRegisterWorthFolding &&
1302 if (isWorthFoldingAddr(RHS,
Size))
1314 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1317 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1319 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1320 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1324bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1354 if (isa<ConstantSDNode>(RHS)) {
1355 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1365 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1368 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1372 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1375 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1376 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1378 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1383 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1384 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1386 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1393 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1394 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1400 static const unsigned RegClassIDs[] = {
1401 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1402 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1403 AArch64::dsub2, AArch64::dsub3};
1409 static const unsigned RegClassIDs[] = {
1410 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1411 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1412 AArch64::qsub2, AArch64::qsub3};
1418 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1419 AArch64::ZPR3RegClassID,
1420 AArch64::ZPR4RegClassID};
1421 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1422 AArch64::zsub2, AArch64::zsub3};
1432 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1433 AArch64::ZPR4Mul4RegClassID};
1434 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1435 AArch64::zsub2, AArch64::zsub3};
1440 const unsigned RegClassIDs[],
1441 const unsigned SubRegs[]) {
1444 if (Regs.
size() == 1)
1455 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1458 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1460 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1464 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1468void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1471 EVT VT =
N->getValueType(0);
1473 unsigned ExtOff = isExt;
1476 unsigned Vec0Off = ExtOff + 1;
1478 N->op_begin() + Vec0Off + NumVecs);
1485 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1486 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1489static std::tuple<SDValue, SDValue>
1509 auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
1510 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1515 AddrDisc = DAG->
getRegister(AArch64::XZR, MVT::i64);
1517 return std::make_tuple(
1522void AArch64DAGToDAGISel::SelectPtrauthAuth(
SDNode *
N) {
1527 SDValue AUTDisc =
N->getOperand(3);
1529 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1530 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1532 SDValue AUTAddrDisc, AUTConstDisc;
1533 std::tie(AUTConstDisc, AUTAddrDisc) =
1536 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1537 AArch64::X16, Val,
SDValue());
1538 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.
getValue(1)};
1540 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT,
DL, MVT::i64, Ops);
1541 ReplaceNode(
N, AUT);
1544void AArch64DAGToDAGISel::SelectPtrauthResign(
SDNode *
N) {
1549 SDValue AUTDisc =
N->getOperand(3);
1551 SDValue PACDisc =
N->getOperand(5);
1553 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1554 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
1556 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1557 PACKey = CurDAG->getTargetConstant(PACKeyC,
DL, MVT::i64);
1559 SDValue AUTAddrDisc, AUTConstDisc;
1560 std::tie(AUTConstDisc, AUTAddrDisc) =
1563 SDValue PACAddrDisc, PACConstDisc;
1564 std::tie(PACConstDisc, PACAddrDisc) =
1567 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1568 AArch64::X16, Val,
SDValue());
1570 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1571 PACConstDisc, PACAddrDisc, X16Copy.
getValue(1)};
1573 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC,
DL, MVT::i64, Ops);
1574 ReplaceNode(
N, AUTPAC);
1577bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1579 if (
LD->isUnindexed())
1581 EVT VT =
LD->getMemoryVT();
1582 EVT DstVT =
N->getValueType(0);
1589 unsigned Opcode = 0;
1592 bool InsertTo64 =
false;
1594 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1595 else if (VT == MVT::i32) {
1597 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1599 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1601 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1607 }
else if (VT == MVT::i16) {
1609 if (DstVT == MVT::i64)
1610 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1612 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1614 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1615 InsertTo64 = DstVT == MVT::i64;
1620 }
else if (VT == MVT::i8) {
1622 if (DstVT == MVT::i64)
1623 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1625 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1627 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1628 InsertTo64 = DstVT == MVT::i64;
1633 }
else if (VT == MVT::f16) {
1634 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1635 }
else if (VT == MVT::bf16) {
1636 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1637 }
else if (VT == MVT::f32) {
1638 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1640 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1642 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1650 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1652 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1657 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1662 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1664 SDValue(CurDAG->getMachineNode(
1665 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1666 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1671 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1674 CurDAG->RemoveDeadNode(
N);
1678void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1679 unsigned SubRegIdx) {
1681 EVT VT =
N->getValueType(0);
1687 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1689 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1691 for (
unsigned i = 0; i < NumVecs; ++i)
1693 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1699 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1701 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1704 CurDAG->RemoveDeadNode(
N);
1707void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1708 unsigned Opc,
unsigned SubRegIdx) {
1710 EVT VT =
N->getValueType(0);
1717 const EVT ResTys[] = {MVT::i64,
1718 MVT::Untyped, MVT::Other};
1720 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1728 ReplaceUses(
SDValue(
N, 0), SuperReg);
1730 for (
unsigned i = 0; i < NumVecs; ++i)
1732 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1736 CurDAG->RemoveDeadNode(
N);
1742std::tuple<unsigned, SDValue, SDValue>
1743AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1749 SDValue NewOffset = OldOffset;
1751 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1752 N, OldBase, NewBase, NewOffset);
1756 const bool IsRegReg =
1757 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1760 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1773template <SelectTypeKind Kind>
1785 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1790 if (EltVT != MVT::i1)
1794 if (EltVT == MVT::bf16)
1796 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1826void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1829 if (
Imm->getZExtValue() > 1)
1833 EVT VT =
N->getValueType(0);
1834 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1835 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1838 for (
unsigned I = 0;
I < 2; ++
I)
1839 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1840 AArch64::psub0 +
I,
DL, VT, SuperReg));
1842 CurDAG->RemoveDeadNode(
N);
1845void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1847 EVT VT =
N->getValueType(0);
1849 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1851 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1854 for (
unsigned I = 0;
I < 2; ++
I)
1855 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1856 AArch64::psub0 +
I,
DL, VT, SuperReg));
1858 CurDAG->RemoveDeadNode(
N);
1861void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1863 EVT VT =
N->getValueType(0);
1865 SDValue Ops = createZTuple(Regs);
1869 for (
unsigned i = 0; i < NumVecs; ++i)
1870 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1871 AArch64::zsub0 + i,
DL, VT, SuperReg));
1873 CurDAG->RemoveDeadNode(
N);
1876void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(
SDNode *
N,
unsigned NumVecs,
1879 EVT VT =
N->getValueType(0);
1881 Ops.push_back(
N->getOperand(0));
1884 CurDAG->getMachineNode(Opcode,
DL, {MVT::Untyped, MVT::Other}, Ops);
1887 for (
unsigned i = 0; i < NumVecs; ++i)
1888 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1889 AArch64::zsub0 + i,
DL, VT, SuperReg));
1892 unsigned ChainIdx = NumVecs;
1894 CurDAG->RemoveDeadNode(
N);
1897void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1902 assert(Opcode != 0 &&
"Unexpected opcode");
1905 EVT VT =
N->getValueType(0);
1906 unsigned FirstVecIdx = HasPred ? 2 : 1;
1908 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1910 return createZMulTuple(Regs);
1913 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1917 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1919 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1923 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1924 N->getOperand(1), Zdn, Zm);
1926 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1928 for (
unsigned i = 0; i < NumVecs; ++i)
1929 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1930 AArch64::zsub0 + i,
DL, VT, SuperReg));
1932 CurDAG->RemoveDeadNode(
N);
1935void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1936 unsigned Scale,
unsigned Opc_ri,
1937 unsigned Opc_rr,
bool IsIntr) {
1938 assert(Scale < 5 &&
"Invalid scaling value.");
1940 EVT VT =
N->getValueType(0);
1946 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1947 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1948 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1950 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1954 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1956 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1958 for (
unsigned i = 0; i < NumVecs; ++i)
1959 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1960 AArch64::zsub0 + i,
DL, VT, SuperReg));
1963 unsigned ChainIdx = NumVecs;
1965 CurDAG->RemoveDeadNode(
N);
1968void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1973 assert(Scale < 4 &&
"Invalid scaling value.");
1975 EVT VT =
N->getValueType(0);
1983 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1989 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1991 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1993 for (
unsigned i = 0; i < NumVecs; ++i)
1994 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1995 AArch64::zsub0 + i,
DL, VT, SuperReg));
1998 unsigned ChainIdx = NumVecs;
2000 CurDAG->RemoveDeadNode(
N);
2003void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
2005 if (
N->getValueType(0) != MVT::nxv4f32)
2007 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
2010void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(
SDNode *
Node,
2011 unsigned NumOutVecs,
2015 if (
Imm->getZExtValue() > MaxImm)
2019 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2024 EVT VT =
Node->getValueType(0);
2027 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
2030 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2031 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
2032 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2035 unsigned ChainIdx = NumOutVecs;
2037 CurDAG->RemoveDeadNode(
Node);
2040void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
2041 unsigned NumOutVecs,
2046 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2052 EVT VT =
Node->getValueType(0);
2055 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
2058 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2059 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
2060 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2063 unsigned ChainIdx = NumOutVecs;
2065 CurDAG->RemoveDeadNode(
Node);
2068void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
2071 EVT VT =
N->getValueType(0);
2074 SDValue Zd = createZMulTuple(Regs);
2075 SDValue Zn =
N->getOperand(1 + NumVecs);
2076 SDValue Zm =
N->getOperand(2 + NumVecs);
2082 for (
unsigned i = 0; i < NumVecs; ++i)
2083 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2084 AArch64::zsub0 + i,
DL, VT, SuperReg));
2086 CurDAG->RemoveDeadNode(
N);
2116template <
unsigned MaxIdx,
unsigned Scale>
2117void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
2118 unsigned BaseReg,
unsigned Op) {
2119 unsigned TileNum = 0;
2120 if (BaseReg != AArch64::ZA)
2121 TileNum =
N->getConstantOperandVal(2);
2127 if (BaseReg == AArch64::ZA)
2128 SliceBase =
N->getOperand(2);
2130 SliceBase =
N->getOperand(3);
2132 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2138 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2140 EVT VT =
N->getValueType(0);
2141 for (
unsigned I = 0;
I < NumVecs; ++
I)
2143 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2146 unsigned ChainIdx = NumVecs;
2148 CurDAG->RemoveDeadNode(
N);
2151void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
2152 unsigned Op,
unsigned MaxIdx,
2153 unsigned Scale,
unsigned BaseReg) {
2157 SDValue SliceBase =
N->getOperand(2);
2158 if (BaseReg != AArch64::ZA)
2159 SliceBase =
N->getOperand(3);
2162 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2169 if (BaseReg != AArch64::ZA )
2174 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2176 EVT VT =
N->getValueType(0);
2177 for (
unsigned I = 0;
I < NumVecs; ++
I)
2179 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2183 unsigned ChainIdx = NumVecs;
2185 CurDAG->RemoveDeadNode(
N);
2188void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2189 unsigned NumOutVecs,
2193 EVT VT =
N->getValueType(0);
2194 unsigned NumInVecs =
N->getNumOperands() - 1;
2198 assert((NumInVecs == 2 || NumInVecs == 4) &&
2199 "Don't know how to handle multi-register input!");
2204 for (
unsigned I = 0;
I < NumInVecs;
I++)
2208 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2211 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2212 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2213 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2214 CurDAG->RemoveDeadNode(
N);
2217void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2220 EVT VT =
N->getOperand(2)->getValueType(0);
2227 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2228 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2232 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2237void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2238 unsigned Scale,
unsigned Opc_rr,
2244 SDValue RegSeq = createZTuple(Regs);
2249 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2250 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2251 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2253 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2257 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2269 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2270 int FI = FINode->getIndex();
2272 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2279void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2282 EVT VT =
N->getOperand(2)->getValueType(0);
2283 const EVT ResTys[] = {MVT::i64,
2292 N->getOperand(NumVecs + 1),
2293 N->getOperand(NumVecs + 2),
2295 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2335void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2338 EVT VT =
N->getValueType(0);
2346 WidenVector(*CurDAG));
2350 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2352 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2354 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2355 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2356 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2360 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2361 AArch64::qsub2, AArch64::qsub3 };
2362 for (
unsigned i = 0; i < NumVecs; ++i) {
2363 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2370 CurDAG->RemoveDeadNode(
N);
2373void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2376 EVT VT =
N->getValueType(0);
2384 WidenVector(*CurDAG));
2388 const EVT ResTys[] = {MVT::i64,
2391 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2394 CurDAG->getTargetConstant(LaneNo, dl,
2396 N->getOperand(NumVecs + 2),
2397 N->getOperand(NumVecs + 3),
2399 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2411 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2412 AArch64::qsub2, AArch64::qsub3 };
2413 for (
unsigned i = 0; i < NumVecs; ++i) {
2414 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2424 CurDAG->RemoveDeadNode(
N);
2427void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2430 EVT VT =
N->getOperand(2)->getValueType(0);
2438 WidenVector(*CurDAG));
2442 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2444 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2445 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2446 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2450 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2455void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2458 EVT VT =
N->getOperand(2)->getValueType(0);
2466 WidenVector(*CurDAG));
2470 const EVT ResTys[] = {MVT::i64,
2473 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2475 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2476 N->getOperand(NumVecs + 2),
2477 N->getOperand(NumVecs + 3),
2479 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2483 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2490 unsigned &LSB,
unsigned &MSB,
2491 unsigned NumberOfIgnoredLowBits,
2492 bool BiggerPattern) {
2494 "N must be a AND operation to call this function");
2496 EVT VT =
N->getValueType(0);
2501 assert((VT == MVT::i32 || VT == MVT::i64) &&
2502 "Type checking must have been done before calling this function");
2516 const SDNode *Op0 =
N->getOperand(0).getNode();
2520 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2523 if (AndImm & (AndImm + 1))
2526 bool ClampMSB =
false;
2546 ClampMSB = (VT == MVT::i32);
2547 }
else if (BiggerPattern) {
2553 Opd0 =
N->getOperand(0);
2559 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2562 <<
": Found large shift immediate, this should not happen\n"));
2568 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2569 : llvm::countr_one<uint64_t>(AndImm)) -
2576 MSB = MSB > 31 ? 31 : MSB;
2578 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2583 SDValue &Opd0,
unsigned &Immr,
2587 EVT VT =
N->getValueType(0);
2589 assert((VT == MVT::i32 || VT == MVT::i64) &&
2590 "Type checking must have been done before calling this function");
2594 Op =
Op->getOperand(0);
2595 VT =
Op->getValueType(0);
2604 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2608 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2609 Opd0 =
Op.getOperand(0);
2611 Imms = ShiftImm + Width - 1;
2639 Opd0 =
N->getOperand(0).getOperand(0);
2649 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2656 unsigned &Immr,
unsigned &Imms,
2657 bool BiggerPattern) {
2659 "N must be a SHR/SRA operation to call this function");
2661 EVT VT =
N->getValueType(0);
2666 assert((VT == MVT::i32 || VT == MVT::i64) &&
2667 "Type checking must have been done before calling this function");
2677 Opd0 =
N->getOperand(0).getOperand(0);
2678 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2684 Opd0 =
N->getOperand(0).getOperand(0);
2687 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2688 }
else if (BiggerPattern) {
2692 Opd0 =
N->getOperand(0);
2701 <<
": Found large shift immediate, this should not happen\n"));
2710 "bad amount in shift node!");
2711 int immr = SrlImm - ShlImm;
2716 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2718 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2722bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2725 EVT VT =
N->getValueType(0);
2726 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2727 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2738 unsigned Immr = ShiftImm;
2740 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2741 CurDAG->getTargetConstant(Imms, dl, VT)};
2742 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2747 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2748 unsigned NumberOfIgnoredLowBits = 0,
2749 bool BiggerPattern =
false) {
2750 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2753 switch (
N->getOpcode()) {
2755 if (!
N->isMachineOpcode())
2760 NumberOfIgnoredLowBits, BiggerPattern);
2769 unsigned NOpc =
N->getMachineOpcode();
2773 case AArch64::SBFMWri:
2774 case AArch64::UBFMWri:
2775 case AArch64::SBFMXri:
2776 case AArch64::UBFMXri:
2778 Opd0 =
N->getOperand(0);
2779 Immr =
N->getConstantOperandVal(1);
2780 Imms =
N->getConstantOperandVal(2);
2787bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2788 unsigned Opc, Immr, Imms;
2793 EVT VT =
N->getValueType(0);
2798 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2799 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2800 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2802 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2803 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2809 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2810 CurDAG->getTargetConstant(Imms, dl, VT)};
2811 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2820 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2821 assert((VT == MVT::i32 || VT == MVT::i64) &&
2822 "i32 or i64 mask type expected!");
2826 APInt SignificantDstMask =
2830 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2831 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2854 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2864 APInt OpUsefulBits(UsefulBits);
2868 OpUsefulBits <<= MSB - Imm + 1;
2873 OpUsefulBits <<= Imm;
2875 OpUsefulBits <<= MSB + 1;
2878 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2884 UsefulBits &= OpUsefulBits;
2890 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2892 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2900 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2901 APInt Mask(UsefulBits);
2902 Mask.clearAllBits();
2910 Mask.lshrInPlace(ShiftAmt);
2916 Mask.lshrInPlace(ShiftAmt);
2928 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2930 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2932 APInt OpUsefulBits(UsefulBits);
2946 OpUsefulBits <<= Width;
2949 if (
Op.getOperand(1) == Orig) {
2951 Mask = ResultUsefulBits & OpUsefulBits;
2955 if (
Op.getOperand(0) == Orig)
2957 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2963 OpUsefulBits <<= Width;
2965 OpUsefulBits <<= LSB;
2967 if (
Op.getOperand(1) == Orig) {
2969 Mask = ResultUsefulBits & OpUsefulBits;
2970 Mask.lshrInPlace(LSB);
2973 if (
Op.getOperand(0) == Orig)
2974 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2991 case AArch64::ANDSWri:
2992 case AArch64::ANDSXri:
2993 case AArch64::ANDWri:
2994 case AArch64::ANDXri:
2998 case AArch64::UBFMWri:
2999 case AArch64::UBFMXri:
3002 case AArch64::ORRWrs:
3003 case AArch64::ORRXrs:
3008 case AArch64::BFMWri:
3009 case AArch64::BFMXri:
3012 case AArch64::STRBBui:
3013 case AArch64::STURBBi:
3019 case AArch64::STRHHui:
3020 case AArch64::STURHHi:
3033 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
3035 UsefulBits =
APInt(Bitwidth, 0);
3044 UsersUsefulBits |= UsefulBitsForUse;
3049 UsefulBits &= UsersUsefulBits;
3059 EVT VT =
Op.getValueType();
3062 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3065 if (ShlAmount > 0) {
3068 UBFMOpc, dl, VT,
Op,
3073 assert(ShlAmount < 0 &&
"expected right shift");
3074 int ShrAmount = -ShlAmount;
3100 bool BiggerPattern,
SDValue &Src,
3101 int &DstLSB,
int &Width) {
3102 EVT VT =
Op.getValueType();
3111 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3115 switch (
Op.getOpcode()) {
3120 NonZeroBits, Src, DstLSB, Width);
3123 NonZeroBits, Src, DstLSB, Width);
3136 EVT VT =
Op.getValueType();
3137 assert((VT == MVT::i32 || VT == MVT::i64) &&
3138 "Caller guarantees VT is one of i32 or i64");
3151 assert((~AndImm & NonZeroBits) == 0 &&
3152 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3181 if (!BiggerPattern && !AndOp0.
hasOneUse())
3200 <<
"Found large Width in bit-field-positioning -- this indicates no "
3201 "proper combining / constant folding was performed\n");
3210 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3225 "Op.getNode() should be a SHL node to call this function");
3227 "Op.getNode() should shift ShlImm to call this function");
3234 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3258 EVT VT =
Op.getValueType();
3259 assert((VT == MVT::i32 || VT == MVT::i64) &&
3260 "Caller guarantees that type is i32 or i64");
3267 if (!BiggerPattern && !
Op.hasOneUse())
3276 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3284 assert(VT == MVT::i32 || VT == MVT::i64);
3295 EVT VT =
N->getValueType(0);
3296 if (VT != MVT::i32 && VT != MVT::i64)
3314 if (!
And.hasOneUse() ||
3324 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3331 if ((OrImm & NotKnownZero) != 0) {
3343 unsigned ImmS = Width - 1;
3349 bool IsBFI = LSB != 0;
3354 unsigned OrChunks = 0, BFIChunks = 0;
3355 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3356 if (((OrImm >> Shift) & 0xFFFF) != 0)
3358 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3361 if (BFIChunks > OrChunks)
3367 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3375 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3384 if (!Dst.hasOneUse())
3387 EVT VT = Dst.getValueType();
3388 assert((VT == MVT::i32 || VT == MVT::i64) &&
3389 "Caller should guarantee that VT is one of i32 or i64");
3397 SDValue DstOp0 = Dst.getOperand(0);
3417 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3418 unsigned MaskWidth =
3421 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3427 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3428 ShiftedOperand =
SDValue(UBFMNode, 0);
3438 ShiftedOperand = Dst.getOperand(0);
3445 ShiftedOperand = Dst.getOperand(0);
3457 const bool BiggerPattern) {
3458 EVT VT =
N->getValueType(0);
3459 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3460 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3461 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3462 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3463 assert((VT == MVT::i32 || VT == MVT::i64) &&
3464 "Expect result type to be i32 or i64 since N is combinable to BFM");
3471 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3474 if (BiggerPattern) {
3488 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3497 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3559 EVT VT =
N->getValueType(0);
3560 if (VT != MVT::i32 && VT != MVT::i64)
3568 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3569 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3589 for (
int I = 0;
I < 4; ++
I) {
3592 unsigned ImmR, ImmS;
3593 bool BiggerPattern =
I / 2;
3594 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3596 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3602 NumberOfIgnoredLowBits, BiggerPattern)) {
3605 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3606 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3611 Width = ImmS - ImmR + 1;
3622 Src, DstLSB, Width)) {
3630 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3640 APInt BitsToBeInserted =
3643 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3667 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3700 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3702 if (Src->hasOneUse() &&
3705 Src = Src->getOperand(0);
3715 unsigned ImmS = Width - 1;
3721 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3729bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3738 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3751bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3755 EVT VT =
N->getValueType(0);
3756 if (VT != MVT::i32 && VT != MVT::i64)
3762 Op0, DstLSB, Width))
3768 unsigned ImmS = Width - 1;
3771 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3772 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3773 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3774 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3780bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3781 EVT VT =
N->getValueType(0);
3784 switch (
N->getOpcode()) {
3786 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3789 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3792 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3795 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3803 if (VT == MVT::i32) {
3806 }
else if (VT == MVT::i64) {
3812 SDValue ShiftAmt =
N->getOperand(1);
3832 (Add0Imm %
Size == 0)) {
3838 if (SubVT == MVT::i32) {
3839 NegOpc = AArch64::SUBWrr;
3840 ZeroReg = AArch64::WZR;
3842 assert(SubVT == MVT::i64);
3843 NegOpc = AArch64::SUBXrr;
3844 ZeroReg = AArch64::XZR;
3847 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3849 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3850 NewShiftAmt =
SDValue(Neg, 0);
3858 if (SubVT == MVT::i32) {
3859 NotOpc = AArch64::ORNWrr;
3860 ZeroReg = AArch64::WZR;
3862 assert(SubVT == MVT::i64);
3863 NotOpc = AArch64::ORNXrr;
3864 ZeroReg = AArch64::XZR;
3867 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3869 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3870 NewShiftAmt =
SDValue(Not, 0);
3891 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3892 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3894 AArch64::SUBREG_TO_REG,
DL, VT,
3895 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3896 NewShiftAmt =
SDValue(Ext, 0);
3899 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3900 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3907 bool isReciprocal) {
3910 FVal = CN->getValueAPF();
3911 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3914 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3918 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3919 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3942 if (!IsExact || !IntVal.isPowerOf2())
3944 unsigned FBits = IntVal.logBase2();
3948 if (FBits == 0 || FBits > RegWidth)
return false;
3954bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3955 unsigned RegWidth) {
3960bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3962 unsigned RegWidth) {
3972 RegString.
split(Fields,
':');
3974 if (Fields.
size() == 1)
3978 &&
"Invalid number of fields in read register string");
3981 bool AllIntFields =
true;
3985 AllIntFields &= !
Field.getAsInteger(10, IntField);
3990 "Unexpected non-integer value in special register string.");
3995 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3996 (Ops[3] << 3) | (Ops[4]);
4003bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
4004 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
4005 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4010 unsigned Opcode64Bit = AArch64::MRS;
4015 const auto *TheReg =
4016 AArch64SysReg::lookupSysRegByName(RegString->getString());
4017 if (TheReg && TheReg->Readable &&
4018 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4019 Imm = TheReg->Encoding;
4025 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
4026 Opcode64Bit = AArch64::ADR;
4034 SDValue InChain =
N->getOperand(0);
4035 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
4036 if (!ReadIs128Bit) {
4037 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
4038 {SysRegImm, InChain});
4042 {MVT::Untyped , MVT::Other },
4043 {SysRegImm, InChain});
4047 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
4049 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
4055 ReplaceUses(
SDValue(
N, 2), OutChain);
4064bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
4065 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
4066 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4071 if (!WriteIs128Bit) {
4077 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
4079 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
4080 "Expected a constant integer expression.");
4081 unsigned Reg = PMapper->Encoding;
4082 uint64_t Immed =
N->getConstantOperandVal(2);
4083 CurDAG->SelectNodeTo(
4084 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
4085 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
4091 if (trySelectPState(
4092 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4093 AArch64::MSRpstateImm4))
4095 if (trySelectPState(
4096 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4097 AArch64::MSRpstateImm1))
4106 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
4107 if (TheReg && TheReg->Writeable &&
4108 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4109 Imm = TheReg->Encoding;
4117 SDValue InChain =
N->getOperand(0);
4118 if (!WriteIs128Bit) {
4119 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
4120 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4121 N->getOperand(2), InChain);
4125 SDNode *Pair = CurDAG->getMachineNode(
4126 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
4127 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
4130 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
4132 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
4134 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
4135 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4143bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
4145 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
4148 if (Subtarget->hasLSE())
return false;
4150 if (MemTy == MVT::i8)
4151 Opcode = AArch64::CMP_SWAP_8;
4152 else if (MemTy == MVT::i16)
4153 Opcode = AArch64::CMP_SWAP_16;
4154 else if (MemTy == MVT::i32)
4155 Opcode = AArch64::CMP_SWAP_32;
4156 else if (MemTy == MVT::i64)
4157 Opcode = AArch64::CMP_SWAP_64;
4161 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4162 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4164 SDNode *CmpSwap = CurDAG->getMachineNode(
4166 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4169 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
4173 CurDAG->RemoveDeadNode(
N);
4180 if (!isa<ConstantSDNode>(
N))
4192 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4193 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4200 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4201 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4205 if (Val <= 65280 && Val % 256 == 0) {
4206 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4207 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4218bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4221 if (!isa<ConstantSDNode>(
N))
4225 int64_t Val = cast<ConstantSDNode>(
N)
4242 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4243 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4250 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4251 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4255 if (Val <= 65280 && Val % 256 == 0) {
4256 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4257 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4270 if (!isa<ConstantSDNode>(
N))
4274 int64_t Val = cast<ConstantSDNode>(
N)
4282 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4283 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4289 if (Val >= -128 && Val <= 127) {
4290 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4291 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4295 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4296 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4297 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4308bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4309 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4310 int64_t ImmVal = CNode->getSExtValue();
4312 if (ImmVal >= -128 && ImmVal < 128) {
4313 Imm = CurDAG->getSignedTargetConstant(ImmVal,
DL, MVT::i32);
4321 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4322 uint64_t ImmVal = CNode->getZExtValue();
4332 ImmVal &= 0xFFFFFFFF;
4341 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4350 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4351 uint64_t ImmVal = CNode->getZExtValue();
4361 ImmVal |= ImmVal << 8;
4362 ImmVal |= ImmVal << 16;
4363 ImmVal |= ImmVal << 32;
4367 ImmVal |= ImmVal << 16;
4368 ImmVal |= ImmVal << 32;
4371 ImmVal &= 0xFFFFFFFF;
4372 ImmVal |= ImmVal << 32;
4382 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4397 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4398 uint64_t ImmVal = CN->getZExtValue();
4405 if (ImmVal >
High) {
4406 if (!AllowSaturation)
4411 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4418bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4422 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4434 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4435 SDValue FiOp = CurDAG->getTargetFrameIndex(
4437 int TagOffset =
N->getConstantOperandVal(3);
4439 SDNode *Out = CurDAG->getMachineNode(
4440 AArch64::TAGPstack,
DL, MVT::i64,
4441 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4442 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4443 ReplaceNode(
N, Out);
4447void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4448 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4449 "llvm.aarch64.tagp third argument must be an immediate");
4450 if (trySelectStackSlotTagP(
N))
4457 int TagOffset =
N->getConstantOperandVal(3);
4458 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4459 {
N->getOperand(1),
N->getOperand(2)});
4460 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4461 {
SDValue(N1, 0),
N->getOperand(2)});
4462 SDNode *N3 = CurDAG->getMachineNode(
4463 AArch64::ADDG,
DL, MVT::i64,
4464 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4465 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4469bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4473 if (
N->getConstantOperandVal(2) != 0)
4475 if (!
N->getOperand(0).isUndef())
4479 EVT VT =
N->getValueType(0);
4480 EVT InVT =
N->getOperand(1).getValueType();
4491 "Expected to insert into a packed scalable vector!");
4494 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4495 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4496 N->getOperand(1), RC));
4500bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4504 if (
N->getConstantOperandVal(1) != 0)
4508 EVT VT =
N->getValueType(0);
4509 EVT InVT =
N->getOperand(0).getValueType();
4520 "Expected to extract from a packed scalable vector!");
4523 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4524 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4525 N->getOperand(0), RC));
4529bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4534 EVT VT =
N->getValueType(0);
4546 (Subtarget->hasSVE2() ||
4547 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4556 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4557 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4564 APInt ShlAmt, ShrAmt;
4577 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4578 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4579 AArch64::XAR_ZZZI_D})) {
4580 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4586 if (!Subtarget->hasSHA3())
4608 if (ShAmt + HsAmt != 64)
4612 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4617void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4619 if (
Node->isMachineOpcode()) {
4621 Node->setNodeId(-1);
4626 EVT VT =
Node->getValueType(0);
4628 switch (
Node->getOpcode()) {
4633 if (SelectCMP_SWAP(
Node))
4639 if (tryReadRegister(
Node))
4645 if (tryWriteRegister(
Node))
4652 if (tryIndexedLoad(
Node))
4661 if (tryBitfieldExtractOp(
Node))
4663 if (tryBitfieldInsertInZeroOp(
Node))
4668 if (tryShiftAmountMod(
Node))
4673 if (tryBitfieldExtractOpFromSExt(
Node))
4678 if (tryBitfieldInsertOp(
Node))
4680 if (trySelectXAR(
Node))
4685 if (trySelectCastScalableToFixedLengthVector(
Node))
4691 if (trySelectCastFixedLengthToScalableVector(
Node))
4700 if (ConstNode->
isZero()) {
4701 if (VT == MVT::i32) {
4703 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4704 ReplaceNode(
Node,
New.getNode());
4706 }
else if (VT == MVT::i64) {
4708 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4709 ReplaceNode(
Node,
New.getNode());
4718 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4721 SDValue TFI = CurDAG->getTargetFrameIndex(
4724 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4725 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4726 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4730 unsigned IntNo =
Node->getConstantOperandVal(1);
4734 case Intrinsic::aarch64_gcsss: {
4738 SDValue Zero = CurDAG->getCopyFromReg(Chain,
DL, AArch64::XZR, MVT::i64);
4740 CurDAG->getMachineNode(AArch64::GCSSS1,
DL, MVT::Other, Val, Chain);
4741 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2,
DL, MVT::i64,
4742 MVT::Other, Zero,
SDValue(SS1, 0));
4743 ReplaceNode(
Node, SS2);
4746 case Intrinsic::aarch64_ldaxp:
4747 case Intrinsic::aarch64_ldxp: {
4749 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4754 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4755 MVT::Other, MemAddr, Chain);
4759 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4760 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4761 ReplaceNode(
Node, Ld);
4764 case Intrinsic::aarch64_stlxp:
4765 case Intrinsic::aarch64_stxp: {
4767 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4775 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4777 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4780 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4781 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4783 ReplaceNode(
Node, St);
4786 case Intrinsic::aarch64_neon_ld1x2:
4787 if (VT == MVT::v8i8) {
4788 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4790 }
else if (VT == MVT::v16i8) {
4791 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4793 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4794 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4796 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4797 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4799 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4800 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4802 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4803 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4805 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4806 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4808 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4809 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4813 case Intrinsic::aarch64_neon_ld1x3:
4814 if (VT == MVT::v8i8) {
4815 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4817 }
else if (VT == MVT::v16i8) {
4818 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4820 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4821 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4823 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4824 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4826 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4827 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4829 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4830 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4832 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4833 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4835 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4836 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4840 case Intrinsic::aarch64_neon_ld1x4:
4841 if (VT == MVT::v8i8) {
4842 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4844 }
else if (VT == MVT::v16i8) {
4845 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4847 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4848 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4850 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4851 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4853 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4854 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4856 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4857 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4859 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4860 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4862 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4863 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4867 case Intrinsic::aarch64_neon_ld2:
4868 if (VT == MVT::v8i8) {
4869 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4871 }
else if (VT == MVT::v16i8) {
4872 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4874 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4875 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4877 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4878 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4880 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4881 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4883 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4884 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4886 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4887 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4889 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4890 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4894 case Intrinsic::aarch64_neon_ld3:
4895 if (VT == MVT::v8i8) {
4896 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4898 }
else if (VT == MVT::v16i8) {
4899 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4901 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4902 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4904 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4905 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4907 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4908 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4910 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4911 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4913 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4914 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4916 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4917 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4921 case Intrinsic::aarch64_neon_ld4:
4922 if (VT == MVT::v8i8) {
4923 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4925 }
else if (VT == MVT::v16i8) {
4926 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4928 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4929 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4931 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4932 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4934 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4935 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4937 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4938 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4940 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4941 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4943 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4944 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4948 case Intrinsic::aarch64_neon_ld2r:
4949 if (VT == MVT::v8i8) {
4950 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4952 }
else if (VT == MVT::v16i8) {
4953 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4955 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4956 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4958 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4959 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4961 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4962 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4964 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4965 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4967 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4968 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4970 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4971 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4975 case Intrinsic::aarch64_neon_ld3r:
4976 if (VT == MVT::v8i8) {
4977 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4979 }
else if (VT == MVT::v16i8) {
4980 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4982 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4983 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4985 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4986 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4988 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4989 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4991 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4992 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4994 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4995 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4997 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4998 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
5002 case Intrinsic::aarch64_neon_ld4r:
5003 if (VT == MVT::v8i8) {
5004 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
5006 }
else if (VT == MVT::v16i8) {
5007 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
5009 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5010 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
5012 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5013 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
5015 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5016 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
5018 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5019 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
5021 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5022 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
5024 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5025 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
5029 case Intrinsic::aarch64_neon_ld2lane:
5030 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5031 SelectLoadLane(
Node, 2, AArch64::LD2i8);
5033 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5034 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5035 SelectLoadLane(
Node, 2, AArch64::LD2i16);
5037 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5039 SelectLoadLane(
Node, 2, AArch64::LD2i32);
5041 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5043 SelectLoadLane(
Node, 2, AArch64::LD2i64);
5047 case Intrinsic::aarch64_neon_ld3lane:
5048 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5049 SelectLoadLane(
Node, 3, AArch64::LD3i8);
5051 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5052 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5053 SelectLoadLane(
Node, 3, AArch64::LD3i16);
5055 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5057 SelectLoadLane(
Node, 3, AArch64::LD3i32);
5059 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5061 SelectLoadLane(
Node, 3, AArch64::LD3i64);
5065 case Intrinsic::aarch64_neon_ld4lane:
5066 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5067 SelectLoadLane(
Node, 4, AArch64::LD4i8);
5069 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5070 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5071 SelectLoadLane(
Node, 4, AArch64::LD4i16);
5073 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5075 SelectLoadLane(
Node, 4, AArch64::LD4i32);
5077 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5079 SelectLoadLane(
Node, 4, AArch64::LD4i64);
5083 case Intrinsic::aarch64_ld64b:
5084 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
5086 case Intrinsic::aarch64_sve_ld2q_sret: {
5087 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
5090 case Intrinsic::aarch64_sve_ld3q_sret: {
5091 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
5094 case Intrinsic::aarch64_sve_ld4q_sret: {
5095 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
5098 case Intrinsic::aarch64_sve_ld2_sret: {
5099 if (VT == MVT::nxv16i8) {
5100 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
5103 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5104 VT == MVT::nxv8bf16) {
5105 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
5108 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5109 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
5112 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5113 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
5119 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5120 if (VT == MVT::nxv16i8) {
5121 if (Subtarget->hasSME2())
5122 SelectContiguousMultiVectorLoad(
5123 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
5124 else if (Subtarget->hasSVE2p1())
5125 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
5130 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5131 VT == MVT::nxv8bf16) {
5132 if (Subtarget->hasSME2())
5133 SelectContiguousMultiVectorLoad(
5134 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
5135 else if (Subtarget->hasSVE2p1())
5136 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
5141 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5142 if (Subtarget->hasSME2())
5143 SelectContiguousMultiVectorLoad(
5144 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
5145 else if (Subtarget->hasSVE2p1())
5146 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
5151 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5152 if (Subtarget->hasSME2())
5153 SelectContiguousMultiVectorLoad(
5154 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
5155 else if (Subtarget->hasSVE2p1())
5156 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
5164 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5165 if (VT == MVT::nxv16i8) {
5166 if (Subtarget->hasSME2())
5167 SelectContiguousMultiVectorLoad(
5168 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5169 else if (Subtarget->hasSVE2p1())
5170 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
5175 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5176 VT == MVT::nxv8bf16) {
5177 if (Subtarget->hasSME2())
5178 SelectContiguousMultiVectorLoad(
5179 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5180 else if (Subtarget->hasSVE2p1())
5181 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
5186 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5187 if (Subtarget->hasSME2())
5188 SelectContiguousMultiVectorLoad(
5189 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5190 else if (Subtarget->hasSVE2p1())
5191 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
5196 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5197 if (Subtarget->hasSME2())
5198 SelectContiguousMultiVectorLoad(
5199 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5200 else if (Subtarget->hasSVE2p1())
5201 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5209 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5210 if (VT == MVT::nxv16i8) {
5211 if (Subtarget->hasSME2())
5212 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5213 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5214 AArch64::LDNT1B_2Z_PSEUDO);
5215 else if (Subtarget->hasSVE2p1())
5216 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5217 AArch64::LDNT1B_2Z);
5221 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5222 VT == MVT::nxv8bf16) {
5223 if (Subtarget->hasSME2())
5224 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5225 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5226 AArch64::LDNT1H_2Z_PSEUDO);
5227 else if (Subtarget->hasSVE2p1())
5228 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5229 AArch64::LDNT1H_2Z);
5233 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5234 if (Subtarget->hasSME2())
5235 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5236 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5237 AArch64::LDNT1W_2Z_PSEUDO);
5238 else if (Subtarget->hasSVE2p1())
5239 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5240 AArch64::LDNT1W_2Z);
5244 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5245 if (Subtarget->hasSME2())
5246 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5247 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5248 AArch64::LDNT1D_2Z_PSEUDO);
5249 else if (Subtarget->hasSVE2p1())
5250 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5251 AArch64::LDNT1D_2Z);
5258 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5259 if (VT == MVT::nxv16i8) {
5260 if (Subtarget->hasSME2())
5261 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5262 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5263 AArch64::LDNT1B_4Z_PSEUDO);
5264 else if (Subtarget->hasSVE2p1())
5265 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5266 AArch64::LDNT1B_4Z);
5270 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5271 VT == MVT::nxv8bf16) {
5272 if (Subtarget->hasSME2())
5273 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5274 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5275 AArch64::LDNT1H_4Z_PSEUDO);
5276 else if (Subtarget->hasSVE2p1())
5277 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5278 AArch64::LDNT1H_4Z);
5282 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5283 if (Subtarget->hasSME2())
5284 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5285 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5286 AArch64::LDNT1W_4Z_PSEUDO);
5287 else if (Subtarget->hasSVE2p1())
5288 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5289 AArch64::LDNT1W_4Z);
5293 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5294 if (Subtarget->hasSME2())
5295 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5296 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5297 AArch64::LDNT1D_4Z_PSEUDO);
5298 else if (Subtarget->hasSVE2p1())
5299 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5300 AArch64::LDNT1D_4Z);
5307 case Intrinsic::aarch64_sve_ld3_sret: {
5308 if (VT == MVT::nxv16i8) {
5309 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5312 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5313 VT == MVT::nxv8bf16) {
5314 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5317 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5318 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5321 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5322 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5328 case Intrinsic::aarch64_sve_ld4_sret: {
5329 if (VT == MVT::nxv16i8) {
5330 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5333 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5334 VT == MVT::nxv8bf16) {
5335 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5338 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5339 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5342 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5343 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5349 case Intrinsic::aarch64_sme_read_hor_vg2: {
5350 if (VT == MVT::nxv16i8) {
5351 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5352 AArch64::MOVA_2ZMXI_H_B);
5354 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5355 VT == MVT::nxv8bf16) {
5356 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5357 AArch64::MOVA_2ZMXI_H_H);
5359 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5360 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5361 AArch64::MOVA_2ZMXI_H_S);
5363 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5364 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5365 AArch64::MOVA_2ZMXI_H_D);
5370 case Intrinsic::aarch64_sme_read_ver_vg2: {
5371 if (VT == MVT::nxv16i8) {
5372 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5373 AArch64::MOVA_2ZMXI_V_B);
5375 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5376 VT == MVT::nxv8bf16) {
5377 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5378 AArch64::MOVA_2ZMXI_V_H);
5380 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5381 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5382 AArch64::MOVA_2ZMXI_V_S);
5384 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5385 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5386 AArch64::MOVA_2ZMXI_V_D);
5391 case Intrinsic::aarch64_sme_read_hor_vg4: {
5392 if (VT == MVT::nxv16i8) {
5393 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5394 AArch64::MOVA_4ZMXI_H_B);
5396 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5397 VT == MVT::nxv8bf16) {
5398 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5399 AArch64::MOVA_4ZMXI_H_H);
5401 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5402 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5403 AArch64::MOVA_4ZMXI_H_S);
5405 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5406 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5407 AArch64::MOVA_4ZMXI_H_D);
5412 case Intrinsic::aarch64_sme_read_ver_vg4: {
5413 if (VT == MVT::nxv16i8) {
5414 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5415 AArch64::MOVA_4ZMXI_V_B);
5417 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5418 VT == MVT::nxv8bf16) {
5419 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5420 AArch64::MOVA_4ZMXI_V_H);
5422 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5423 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5424 AArch64::MOVA_4ZMXI_V_S);
5426 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5427 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5428 AArch64::MOVA_4ZMXI_V_D);
5433 case Intrinsic::aarch64_sme_read_vg1x2: {
5434 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5435 AArch64::MOVA_VG2_2ZMXI);
5438 case Intrinsic::aarch64_sme_read_vg1x4: {
5439 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5440 AArch64::MOVA_VG4_4ZMXI);
5443 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5444 if (VT == MVT::nxv16i8) {
5445 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5447 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5448 VT == MVT::nxv8bf16) {
5449 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5451 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5452 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5454 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5455 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5460 case Intrinsic::aarch64_sme_readz_vert_x2: {
5461 if (VT == MVT::nxv16i8) {
5462 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
5464 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5465 VT == MVT::nxv8bf16) {
5466 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
5468 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5469 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
5471 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5472 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
5477 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5478 if (VT == MVT::nxv16i8) {
5479 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
5481 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5482 VT == MVT::nxv8bf16) {
5483 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
5485 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5486 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
5488 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5489 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
5494 case Intrinsic::aarch64_sme_readz_vert_x4: {
5495 if (VT == MVT::nxv16i8) {
5496 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
5498 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5499 VT == MVT::nxv8bf16) {
5500 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
5502 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5503 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
5505 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5506 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
5511 case Intrinsic::aarch64_sme_readz_x2: {
5512 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, 7, 1,
5516 case Intrinsic::aarch64_sme_readz_x4: {
5517 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, 7, 1,
5521 case Intrinsic::swift_async_context_addr: {
5524 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5526 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5527 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5528 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5532 CurDAG->RemoveDeadNode(
Node);
5534 auto &MF = CurDAG->getMachineFunction();
5535 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5539 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5540 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5541 Node->getValueType(0),
5542 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5543 AArch64::LUTI2_4ZTZI_S}))
5545 SelectMultiVectorLutiLane(
Node, 4, Opc, 3);
5548 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5549 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5550 Node->getValueType(0),
5551 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5553 SelectMultiVectorLutiLane(
Node, 4, Opc, 1);
5556 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5557 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5558 Node->getValueType(0),
5559 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5560 AArch64::LUTI2_2ZTZI_S}))
5562 SelectMultiVectorLutiLane(
Node, 2, Opc, 7);
5565 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5566 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5567 Node->getValueType(0),
5568 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5569 AArch64::LUTI4_2ZTZI_S}))
5571 SelectMultiVectorLutiLane(
Node, 2, Opc, 3);
5574 case Intrinsic::aarch64_sme_luti4_zt_x4: {
5575 SelectMultiVectorLuti(
Node, 4, AArch64::LUTI4_4ZZT2Z);
5578 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
5579 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
5580 Node->getValueType(0),
5581 {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
5582 SelectCVTIntrinsicFP8(
Node, 2, Opc);
5584 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
5585 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
5586 Node->getValueType(0),
5587 {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
5588 SelectCVTIntrinsicFP8(
Node, 2, Opc);
5590 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
5591 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
5592 Node->getValueType(0),
5593 {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
5594 SelectCVTIntrinsicFP8(
Node, 2, Opc);
5596 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
5597 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
5598 Node->getValueType(0),
5599 {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
5600 SelectCVTIntrinsicFP8(
Node, 2, Opc);
5605 unsigned IntNo =
Node->getConstantOperandVal(0);
5609 case Intrinsic::aarch64_tagp:
5613 case Intrinsic::ptrauth_auth:
5614 SelectPtrauthAuth(
Node);
5617 case Intrinsic::ptrauth_resign:
5618 SelectPtrauthResign(
Node);
5621 case Intrinsic::aarch64_neon_tbl2:
5622 SelectTable(
Node, 2,
5623 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5626 case Intrinsic::aarch64_neon_tbl3:
5627 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5628 : AArch64::TBLv16i8Three,
5631 case Intrinsic::aarch64_neon_tbl4:
5632 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5633 : AArch64::TBLv16i8Four,
5636 case Intrinsic::aarch64_neon_tbx2:
5637 SelectTable(
Node, 2,
5638 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5641 case Intrinsic::aarch64_neon_tbx3:
5642 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5643 : AArch64::TBXv16i8Three,
5646 case Intrinsic::aarch64_neon_tbx4:
5647 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5648 : AArch64::TBXv16i8Four,
5651 case Intrinsic::aarch64_sve_srshl_single_x2:
5652 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5653 Node->getValueType(0),
5654 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5655 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5656 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5658 case Intrinsic::aarch64_sve_srshl_single_x4:
5659 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5660 Node->getValueType(0),
5661 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5662 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5663 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5665 case Intrinsic::aarch64_sve_urshl_single_x2:
5666 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5667 Node->getValueType(0),
5668 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5669 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5670 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5672 case Intrinsic::aarch64_sve_urshl_single_x4:
5673 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5674 Node->getValueType(0),
5675 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5676 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5677 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5679 case Intrinsic::aarch64_sve_srshl_x2:
5680 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5681 Node->getValueType(0),
5682 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5683 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5684 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5686 case Intrinsic::aarch64_sve_srshl_x4:
5687 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5688 Node->getValueType(0),
5689 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5690 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5691 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5693 case Intrinsic::aarch64_sve_urshl_x2:
5694 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5695 Node->getValueType(0),
5696 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5697 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5698 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5700 case Intrinsic::aarch64_sve_urshl_x4:
5701 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5702 Node->getValueType(0),
5703 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5704 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5705 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5707 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5708 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5709 Node->getValueType(0),
5710 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5711 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5712 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5714 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5715 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5716 Node->getValueType(0),
5717 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5718 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5719 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5721 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5722 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5723 Node->getValueType(0),
5724 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5725 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5726 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5728 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5729 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5730 Node->getValueType(0),
5731 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5732 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5733 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5735 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
5736 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5737 Node->getValueType(0),
5738 {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
5739 AArch64::FSCALE_2ZZ_D}))
5740 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5742 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
5743 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5744 Node->getValueType(0),
5745 {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
5746 AArch64::FSCALE_4ZZ_D}))
5747 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5749 case Intrinsic::aarch64_sme_fp8_scale_x2:
5750 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5751 Node->getValueType(0),
5752 {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
5753 AArch64::FSCALE_2Z2Z_D}))
5754 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5756 case Intrinsic::aarch64_sme_fp8_scale_x4:
5757 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5758 Node->getValueType(0),
5759 {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
5760 AArch64::FSCALE_4Z4Z_D}))
5761 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5763 case Intrinsic::aarch64_sve_whilege_x2:
5764 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5765 Node->getValueType(0),
5766 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5767 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5768 SelectWhilePair(
Node,
Op);
5770 case Intrinsic::aarch64_sve_whilegt_x2:
5771 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5772 Node->getValueType(0),
5773 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5774 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5775 SelectWhilePair(
Node,
Op);
5777 case Intrinsic::aarch64_sve_whilehi_x2:
5778 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5779 Node->getValueType(0),
5780 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5781 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5782 SelectWhilePair(
Node,
Op);
5784 case Intrinsic::aarch64_sve_whilehs_x2:
5785 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5786 Node->getValueType(0),
5787 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5788 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5789 SelectWhilePair(
Node,
Op);
5791 case Intrinsic::aarch64_sve_whilele_x2:
5792 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5793 Node->getValueType(0),
5794 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5795 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5796 SelectWhilePair(
Node,
Op);
5798 case Intrinsic::aarch64_sve_whilelo_x2:
5799 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5800 Node->getValueType(0),
5801 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5802 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5803 SelectWhilePair(
Node,
Op);
5805 case Intrinsic::aarch64_sve_whilels_x2:
5806 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5807 Node->getValueType(0),
5808 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5809 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5810 SelectWhilePair(
Node,
Op);
5812 case Intrinsic::aarch64_sve_whilelt_x2:
5813 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5814 Node->getValueType(0),
5815 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5816 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5817 SelectWhilePair(
Node,
Op);
5819 case Intrinsic::aarch64_sve_smax_single_x2:
5820 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5821 Node->getValueType(0),
5822 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5823 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5824 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5826 case Intrinsic::aarch64_sve_umax_single_x2:
5827 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5828 Node->getValueType(0),
5829 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5830 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5831 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5833 case Intrinsic::aarch64_sve_fmax_single_x2:
5834 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5835 Node->getValueType(0),
5836 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
5837 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
5838 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5840 case Intrinsic::aarch64_sve_smax_single_x4:
5841 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5842 Node->getValueType(0),
5843 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5844 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5845 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5847 case Intrinsic::aarch64_sve_umax_single_x4:
5848 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5849 Node->getValueType(0),
5850 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5851 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5852 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5854 case Intrinsic::aarch64_sve_fmax_single_x4:
5855 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5856 Node->getValueType(0),
5857 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
5858 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
5859 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5861 case Intrinsic::aarch64_sve_smin_single_x2:
5862 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5863 Node->getValueType(0),
5864 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5865 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5866 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5868 case Intrinsic::aarch64_sve_umin_single_x2:
5869 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5870 Node->getValueType(0),
5871 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5872 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5873 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5875 case Intrinsic::aarch64_sve_fmin_single_x2:
5876 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5877 Node->getValueType(0),
5878 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
5879 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
5880 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5882 case Intrinsic::aarch64_sve_smin_single_x4:
5883 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5884 Node->getValueType(0),
5885 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5886 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5887 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5889 case Intrinsic::aarch64_sve_umin_single_x4:
5890 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5891 Node->getValueType(0),
5892 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5893 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5894 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5896 case Intrinsic::aarch64_sve_fmin_single_x4:
5897 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5898 Node->getValueType(0),
5899 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
5900 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
5901 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5903 case Intrinsic::aarch64_sve_smax_x2:
5904 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5905 Node->getValueType(0),
5906 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5907 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5908 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5910 case Intrinsic::aarch64_sve_umax_x2:
5911 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5912 Node->getValueType(0),
5913 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5914 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5915 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5917 case Intrinsic::aarch64_sve_fmax_x2:
5918 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5919 Node->getValueType(0),
5920 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
5921 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
5922 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5924 case Intrinsic::aarch64_sve_smax_x4:
5925 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5926 Node->getValueType(0),
5927 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5928 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5929 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5931 case Intrinsic::aarch64_sve_umax_x4:
5932 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5933 Node->getValueType(0),
5934 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5935 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5936 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5938 case Intrinsic::aarch64_sve_fmax_x4:
5939 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5940 Node->getValueType(0),
5941 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
5942 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
5943 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5945 case Intrinsic::aarch64_sme_famax_x2:
5946 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5947 Node->getValueType(0),
5948 {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
5949 AArch64::FAMAX_2Z2Z_D}))
5950 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5952 case Intrinsic::aarch64_sme_famax_x4:
5953 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5954 Node->getValueType(0),
5955 {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
5956 AArch64::FAMAX_4Z4Z_D}))
5957 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5959 case Intrinsic::aarch64_sme_famin_x2:
5960 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5961 Node->getValueType(0),
5962 {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
5963 AArch64::FAMIN_2Z2Z_D}))
5964 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5966 case Intrinsic::aarch64_sme_famin_x4:
5967 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5968 Node->getValueType(0),
5969 {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
5970 AArch64::FAMIN_4Z4Z_D}))
5971 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5973 case Intrinsic::aarch64_sve_smin_x2:
5974 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5975 Node->getValueType(0),
5976 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5977 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5978 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5980 case Intrinsic::aarch64_sve_umin_x2:
5981 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5982 Node->getValueType(0),
5983 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5984 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5985 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5987 case Intrinsic::aarch64_sve_fmin_x2:
5988 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5989 Node->getValueType(0),
5990 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
5991 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
5992 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5994 case Intrinsic::aarch64_sve_smin_x4:
5995 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5996 Node->getValueType(0),
5997 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5998 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5999 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
6001 case Intrinsic::aarch64_sve_umin_x4:
6002 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6003 Node->getValueType(0),
6004 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6005 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6006 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
6008 case Intrinsic::aarch64_sve_fmin_x4:
6009 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6010 Node->getValueType(0),
6011 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6012 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6013 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
6015 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6016 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6017 Node->getValueType(0),
6018 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6019 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6020 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
6022 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6023 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6024 Node->getValueType(0),
6025 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6026 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6027 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
6029 case Intrinsic::aarch64_sve_fminnm_single_x2:
6030 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6031 Node->getValueType(0),
6032 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6033 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6034 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
6036 case Intrinsic::aarch64_sve_fminnm_single_x4:
6037 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6038 Node->getValueType(0),
6039 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6040 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6041 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
6043 case Intrinsic::aarch64_sve_fmaxnm_x2:
6044 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6045 Node->getValueType(0),
6046 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6047 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6048 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
6050 case Intrinsic::aarch64_sve_fmaxnm_x4:
6051 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6052 Node->getValueType(0),
6053 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6054 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6055 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
6057 case Intrinsic::aarch64_sve_fminnm_x2:
6058 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6059 Node->getValueType(0),
6060 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6061 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6062 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
6064 case Intrinsic::aarch64_sve_fminnm_x4:
6065 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6066 Node->getValueType(0),
6067 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6068 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6069 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
6071 case Intrinsic::aarch64_sve_fcvtzs_x2:
6072 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
6074 case Intrinsic::aarch64_sve_scvtf_x2:
6075 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
6077 case Intrinsic::aarch64_sve_fcvtzu_x2:
6078 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
6080 case Intrinsic::aarch64_sve_ucvtf_x2:
6081 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
6083 case Intrinsic::aarch64_sve_fcvtzs_x4:
6084 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
6086 case Intrinsic::aarch64_sve_scvtf_x4:
6087 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
6089 case Intrinsic::aarch64_sve_fcvtzu_x4:
6090 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
6092 case Intrinsic::aarch64_sve_ucvtf_x4:
6093 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
6095 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6096 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVT_2ZZ_H_S);
6098 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6099 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVTL_2ZZ_H_S);
6101 case Intrinsic::aarch64_sve_sclamp_single_x2:
6102 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6103 Node->getValueType(0),
6104 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6105 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6106 SelectClamp(
Node, 2,
Op);
6108 case Intrinsic::aarch64_sve_uclamp_single_x2:
6109 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6110 Node->getValueType(0),
6111 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6112 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6113 SelectClamp(
Node, 2,
Op);
6115 case Intrinsic::aarch64_sve_fclamp_single_x2:
6116 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6117 Node->getValueType(0),
6118 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6119 AArch64::FCLAMP_VG2_2Z2Z_D}))
6120 SelectClamp(
Node, 2,
Op);
6122 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6123 SelectClamp(
Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
6125 case Intrinsic::aarch64_sve_sclamp_single_x4:
6126 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6127 Node->getValueType(0),
6128 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6129 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6130 SelectClamp(
Node, 4,
Op);
6132 case Intrinsic::aarch64_sve_uclamp_single_x4:
6133 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6134 Node->getValueType(0),
6135 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6136 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6137 SelectClamp(
Node, 4,
Op);
6139 case Intrinsic::aarch64_sve_fclamp_single_x4:
6140 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6141 Node->getValueType(0),
6142 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6143 AArch64::FCLAMP_VG4_4Z4Z_D}))
6144 SelectClamp(
Node, 4,
Op);
6146 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6147 SelectClamp(
Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
6149 case Intrinsic::aarch64_sve_add_single_x2:
6150 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6151 Node->getValueType(0),
6152 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6153 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6154 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
6156 case Intrinsic::aarch64_sve_add_single_x4:
6157 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6158 Node->getValueType(0),
6159 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6160 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6161 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
6163 case Intrinsic::aarch64_sve_zip_x2:
6164 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6165 Node->getValueType(0),
6166 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6167 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6168 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6170 case Intrinsic::aarch64_sve_zipq_x2:
6171 SelectUnaryMultiIntrinsic(
Node, 2,
false,
6172 AArch64::ZIP_VG2_2ZZZ_Q);
6174 case Intrinsic::aarch64_sve_zip_x4:
6175 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6176 Node->getValueType(0),
6177 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6178 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6179 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6181 case Intrinsic::aarch64_sve_zipq_x4:
6182 SelectUnaryMultiIntrinsic(
Node, 4,
true,
6183 AArch64::ZIP_VG4_4Z4Z_Q);
6185 case Intrinsic::aarch64_sve_uzp_x2:
6186 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6187 Node->getValueType(0),
6188 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6189 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6190 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6192 case Intrinsic::aarch64_sve_uzpq_x2:
6193 SelectUnaryMultiIntrinsic(
Node, 2,
false,
6194 AArch64::UZP_VG2_2ZZZ_Q);
6196 case Intrinsic::aarch64_sve_uzp_x4:
6197 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6198 Node->getValueType(0),
6199 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6200 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6201 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6203 case Intrinsic::aarch64_sve_uzpq_x4:
6204 SelectUnaryMultiIntrinsic(
Node, 4,
true,
6205 AArch64::UZP_VG4_4Z4Z_Q);
6207 case Intrinsic::aarch64_sve_sel_x2:
6208 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6209 Node->getValueType(0),
6210 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6211 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6212 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
6214 case Intrinsic::aarch64_sve_sel_x4:
6215 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6216 Node->getValueType(0),
6217 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6218 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6219 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
6221 case Intrinsic::aarch64_sve_frinta_x2:
6222 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
6224 case Intrinsic::aarch64_sve_frinta_x4:
6225 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
6227 case Intrinsic::aarch64_sve_frintm_x2:
6228 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
6230 case Intrinsic::aarch64_sve_frintm_x4:
6231 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
6233 case Intrinsic::aarch64_sve_frintn_x2:
6234 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
6236 case Intrinsic::aarch64_sve_frintn_x4:
6237 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
6239 case Intrinsic::aarch64_sve_frintp_x2:
6240 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
6242 case Intrinsic::aarch64_sve_frintp_x4:
6243 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
6245 case Intrinsic::aarch64_sve_sunpk_x2:
6246 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6247 Node->getValueType(0),
6248 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6249 AArch64::SUNPK_VG2_2ZZ_D}))
6250 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6252 case Intrinsic::aarch64_sve_uunpk_x2:
6253 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6254 Node->getValueType(0),
6255 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6256 AArch64::UUNPK_VG2_2ZZ_D}))
6257 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6259 case Intrinsic::aarch64_sve_sunpk_x4:
6260 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6261 Node->getValueType(0),
6262 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6263 AArch64::SUNPK_VG4_4Z2Z_D}))
6264 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6266 case Intrinsic::aarch64_sve_uunpk_x4:
6267 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6268 Node->getValueType(0),
6269 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6270 AArch64::UUNPK_VG4_4Z2Z_D}))
6271 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6273 case Intrinsic::aarch64_sve_pext_x2: {
6274 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6275 Node->getValueType(0),
6276 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6277 AArch64::PEXT_2PCI_D}))
6278 SelectPExtPair(
Node,
Op);
6285 unsigned IntNo =
Node->getConstantOperandVal(1);
6286 if (
Node->getNumOperands() >= 3)
6287 VT =
Node->getOperand(2)->getValueType(0);
6291 case Intrinsic::aarch64_neon_st1x2: {
6292 if (VT == MVT::v8i8) {
6293 SelectStore(
Node, 2, AArch64::ST1Twov8b);
6295 }
else if (VT == MVT::v16i8) {
6296 SelectStore(
Node, 2, AArch64::ST1Twov16b);
6298 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6299 VT == MVT::v4bf16) {
6300 SelectStore(
Node, 2, AArch64::ST1Twov4h);
6302 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6303 VT == MVT::v8bf16) {
6304 SelectStore(
Node, 2, AArch64::ST1Twov8h);
6306 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6307 SelectStore(
Node, 2, AArch64::ST1Twov2s);
6309 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6310 SelectStore(
Node, 2, AArch64::ST1Twov4s);
6312 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6313 SelectStore(
Node, 2, AArch64::ST1Twov2d);
6315 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6316 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6321 case Intrinsic::aarch64_neon_st1x3: {
6322 if (VT == MVT::v8i8) {
6323 SelectStore(
Node, 3, AArch64::ST1Threev8b);
6325 }
else if (VT == MVT::v16i8) {
6326 SelectStore(
Node, 3, AArch64::ST1Threev16b);
6328 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6329 VT == MVT::v4bf16) {
6330 SelectStore(
Node, 3, AArch64::ST1Threev4h);
6332 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6333 VT == MVT::v8bf16) {
6334 SelectStore(
Node, 3, AArch64::ST1Threev8h);
6336 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6337 SelectStore(
Node, 3, AArch64::ST1Threev2s);
6339 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6340 SelectStore(
Node, 3, AArch64::ST1Threev4s);
6342 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6343 SelectStore(
Node, 3, AArch64::ST1Threev2d);
6345 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6346 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6351 case Intrinsic::aarch64_neon_st1x4: {
6352 if (VT == MVT::v8i8) {
6353 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
6355 }
else if (VT == MVT::v16i8) {
6356 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
6358 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6359 VT == MVT::v4bf16) {
6360 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
6362 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6363 VT == MVT::v8bf16) {
6364 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
6366 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6367 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
6369 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6370 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
6372 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6373 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
6375 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6376 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6381 case Intrinsic::aarch64_neon_st2: {
6382 if (VT == MVT::v8i8) {
6383 SelectStore(
Node, 2, AArch64::ST2Twov8b);
6385 }
else if (VT == MVT::v16i8) {
6386 SelectStore(
Node, 2, AArch64::ST2Twov16b);
6388 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6389 VT == MVT::v4bf16) {
6390 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6392 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6393 VT == MVT::v8bf16) {
6394 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6396 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6397 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6399 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6400 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6402 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6403 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6405 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6406 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6411 case Intrinsic::aarch64_neon_st3: {
6412 if (VT == MVT::v8i8) {
6413 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6415 }
else if (VT == MVT::v16i8) {
6416 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6418 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6419 VT == MVT::v4bf16) {
6420 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6422 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6423 VT == MVT::v8bf16) {
6424 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6426 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6427 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6429 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6430 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6432 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6433 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6435 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6436 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6441 case Intrinsic::aarch64_neon_st4: {
6442 if (VT == MVT::v8i8) {
6443 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6445 }
else if (VT == MVT::v16i8) {
6446 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6448 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6449 VT == MVT::v4bf16) {
6450 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6452 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6453 VT == MVT::v8bf16) {
6454 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6456 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6457 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6459 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6460 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6462 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6463 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6465 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6466 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6471 case Intrinsic::aarch64_neon_st2lane: {
6472 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6473 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6475 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6476 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6477 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6479 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6481 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6483 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6485 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6490 case Intrinsic::aarch64_neon_st3lane: {
6491 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6492 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6494 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6495 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6496 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6498 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6500 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6502 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6504 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6509 case Intrinsic::aarch64_neon_st4lane: {
6510 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6511 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6513 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6514 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6515 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6517 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6519 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6521 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6523 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6528 case Intrinsic::aarch64_sve_st2q: {
6529 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6532 case Intrinsic::aarch64_sve_st3q: {
6533 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6536 case Intrinsic::aarch64_sve_st4q: {
6537 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6540 case Intrinsic::aarch64_sve_st2: {
6541 if (VT == MVT::nxv16i8) {
6542 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6544 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6545 VT == MVT::nxv8bf16) {
6546 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6548 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6549 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6551 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6552 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6557 case Intrinsic::aarch64_sve_st3: {
6558 if (VT == MVT::nxv16i8) {
6559 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6561 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6562 VT == MVT::nxv8bf16) {
6563 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6565 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6566 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6568 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6569 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6574 case Intrinsic::aarch64_sve_st4: {
6575 if (VT == MVT::nxv16i8) {
6576 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6578 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6579 VT == MVT::nxv8bf16) {
6580 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6582 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6583 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6585 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6586 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6595 if (VT == MVT::v8i8) {
6596 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6598 }
else if (VT == MVT::v16i8) {
6599 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6601 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6602 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6604 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6605 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6607 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6608 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6610 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6611 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6613 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6614 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6616 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6617 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6623 if (VT == MVT::v8i8) {
6624 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6626 }
else if (VT == MVT::v16i8) {
6627 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6629 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6630 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6632 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6633 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6635 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6636 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6638 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6639 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6641 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6642 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6644 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6645 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6651 if (VT == MVT::v8i8) {
6652 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6654 }
else if (VT == MVT::v16i8) {
6655 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6657 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6658 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6660 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6661 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6663 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6664 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6666 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6667 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6669 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6670 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6672 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6673 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6679 if (VT == MVT::v8i8) {
6680 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6682 }
else if (VT == MVT::v16i8) {
6683 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6685 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6686 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6688 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6689 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6691 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6692 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6694 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6695 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6697 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6698 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6700 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6701 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6707 if (VT == MVT::v8i8) {
6708 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6710 }
else if (VT == MVT::v16i8) {
6711 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6713 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6714 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6716 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6717 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6719 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6720 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6722 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6723 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6725 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6726 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6728 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6729 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6735 if (VT == MVT::v8i8) {
6736 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6738 }
else if (VT == MVT::v16i8) {
6739 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6741 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6742 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6744 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6745 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6747 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6748 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6750 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6751 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6753 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6754 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6756 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6757 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6763 if (VT == MVT::v8i8) {
6764 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6766 }
else if (VT == MVT::v16i8) {
6767 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6769 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6770 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6772 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6773 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6775 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6776 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6778 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6779 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6781 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6782 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6784 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6785 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6791 if (VT == MVT::v8i8) {
6792 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6794 }
else if (VT == MVT::v16i8) {
6795 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6797 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6798 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6800 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6801 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6803 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6804 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6806 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6807 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6809 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6810 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6812 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6813 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6819 if (VT == MVT::v8i8) {
6820 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6822 }
else if (VT == MVT::v16i8) {
6823 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6825 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6826 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6828 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6829 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6831 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6832 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6834 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6835 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6837 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6838 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6840 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6841 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6847 if (VT == MVT::v8i8) {
6848 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6850 }
else if (VT == MVT::v16i8) {
6851 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6853 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6854 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6856 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6857 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6859 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6860 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6862 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6863 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6865 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6866 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6868 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6869 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6875 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6876 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6878 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6879 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6880 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6882 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6884 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6886 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6888 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6894 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6895 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6897 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6898 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6899 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6901 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6903 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6905 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6907 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6913 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6914 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6916 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6917 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6918 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6920 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6922 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6924 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6926 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6932 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6933 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6935 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6936 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6937 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6939 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6941 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6943 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6945 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6951 VT =
Node->getOperand(1).getValueType();
6952 if (VT == MVT::v8i8) {
6953 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6955 }
else if (VT == MVT::v16i8) {
6956 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6958 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6959 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6961 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6962 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6964 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6965 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6967 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6968 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6970 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6971 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6973 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6974 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6980 VT =
Node->getOperand(1).getValueType();
6981 if (VT == MVT::v8i8) {
6982 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6984 }
else if (VT == MVT::v16i8) {
6985 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6987 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6988 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6990 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6991 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6993 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6994 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6996 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6997 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6999 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7000 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
7002 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7003 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
7009 VT =
Node->getOperand(1).getValueType();
7010 if (VT == MVT::v8i8) {
7011 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
7013 }
else if (VT == MVT::v16i8) {
7014 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
7016 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7017 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
7019 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7020 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
7022 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7023 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
7025 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7026 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
7028 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7029 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
7031 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7032 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
7038 VT =
Node->getOperand(1).getValueType();
7039 if (VT == MVT::v8i8) {
7040 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
7042 }
else if (VT == MVT::v16i8) {
7043 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
7045 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7046 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
7048 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7049 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
7051 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7052 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
7054 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7055 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
7057 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7058 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
7060 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7061 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
7067 VT =
Node->getOperand(1).getValueType();
7068 if (VT == MVT::v8i8) {
7069 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
7071 }
else if (VT == MVT::v16i8) {
7072 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
7074 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7075 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
7077 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7078 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
7080 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7081 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
7083 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7084 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
7086 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7087 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
7089 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7090 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
7096 VT =
Node->getOperand(1).getValueType();
7097 if (VT == MVT::v8i8) {
7098 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
7100 }
else if (VT == MVT::v16i8) {
7101 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
7103 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7104 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
7106 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7107 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
7109 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7110 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
7112 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7113 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
7115 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7116 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
7118 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7119 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
7125 VT =
Node->getOperand(1).getValueType();
7126 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7127 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
7129 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7130 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7131 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
7133 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7135 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
7137 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7139 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
7145 VT =
Node->getOperand(1).getValueType();
7146 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7147 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
7149 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7150 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7151 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
7153 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7155 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
7157 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7159 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
7165 VT =
Node->getOperand(1).getValueType();
7166 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7167 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
7169 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7170 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7171 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
7173 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7175 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
7177 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7179 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
7185 if (VT == MVT::nxv16i8) {
7186 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
7188 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7189 VT == MVT::nxv8bf16) {
7190 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
7192 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7193 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
7195 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7196 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
7202 if (VT == MVT::nxv16i8) {
7203 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
7205 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7206 VT == MVT::nxv8bf16) {
7207 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
7209 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7210 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
7212 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7213 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
7219 if (VT == MVT::nxv16i8) {
7220 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
7222 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7223 VT == MVT::nxv8bf16) {
7224 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
7226 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7227 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
7229 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7230 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
7245 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7257 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
7261 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7262 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7276 if (isa<MemSDNode>(Root))
7277 return cast<MemSDNode>(Root)->getMemoryVT();
7279 if (isa<MemIntrinsicSDNode>(Root))
7280 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
7282 const unsigned Opcode = Root->
getOpcode();
7290 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
7292 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
7312 case Intrinsic::aarch64_sme_ldr:
7313 case Intrinsic::aarch64_sme_str:
7314 return MVT::nxv16i8;
7315 case Intrinsic::aarch64_sve_prf:
7320 case Intrinsic::aarch64_sve_ld2_sret:
7321 case Intrinsic::aarch64_sve_ld2q_sret:
7324 case Intrinsic::aarch64_sve_st2q:
7327 case Intrinsic::aarch64_sve_ld3_sret:
7328 case Intrinsic::aarch64_sve_ld3q_sret:
7331 case Intrinsic::aarch64_sve_st3q:
7334 case Intrinsic::aarch64_sve_ld4_sret:
7335 case Intrinsic::aarch64_sve_ld4q_sret:
7338 case Intrinsic::aarch64_sve_st4q:
7341 case Intrinsic::aarch64_sve_ld1udq:
7342 case Intrinsic::aarch64_sve_st1dq:
7343 return EVT(MVT::nxv1i64);
7344 case Intrinsic::aarch64_sve_ld1uwq:
7345 case Intrinsic::aarch64_sve_st1wq:
7346 return EVT(MVT::nxv1i32);
7353template <
int64_t Min,
int64_t Max>
7354bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
7362 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
7367 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
7386 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
7388 if ((MulImm % MemWidthBytes) != 0)
7391 int64_t
Offset = MulImm / MemWidthBytes;
7392 if (Offset < Min || Offset > Max)
7395 Base =
N.getOperand(0);
7397 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7404 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7410bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7428 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7429 int64_t ImmOff =
C->getSExtValue();
7430 unsigned Size = 1 << Scale;
7439 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7441 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7451 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7452 if (
C->getZExtValue() == Scale) {
7461bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7468bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7469 EVT VT =
N.getValueType();
7473bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7477 if (CurDAG->isBaseWithConstantOffset(
N))
7478 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7479 int64_t ImmOff =
C->getSExtValue();
7480 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7481 Base =
N.getOperand(0);
7482 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7489 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static bool isMemOpOrPrefetch(SDNode *N)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
unsigned const TargetRegisterInfo * TRI
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SDValue getRegister(Register Reg, EVT VT)
static constexpr unsigned MaxRecursionDepth
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Not(const Pred &P) -> Not< Pred >
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.