32#include "llvm/IR/IntrinsicsARM.h"
42#define DEBUG_TYPE "arm-isel"
43#define PASS_NAME "ARM Instruction Selection"
47 cl::desc(
"Disable isel of shifter-op"),
62 ARMDAGToDAGISel() =
delete;
78 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &dl) {
79 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
88 bool hasNoVMLxHazardUse(
SDNode *
N)
const;
89 bool isShifterOpProfitable(
const SDValue &Shift,
93 bool CheckProfitability =
true);
95 SDValue &
B,
bool CheckProfitability =
true);
99 return SelectRegShifterOperand(
N,
A,
B,
C,
false);
103 return SelectImmShifterOperand(
N,
A,
B,
false);
108 return SelectImmShifterOperand(
N,
A,
B,
false);
119 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
154 template <
unsigned Shift>
159 template <
unsigned Shift>
165 template <
unsigned Shift>
169 template <
unsigned Shift>
175 template<
int Min,
int Max>
178 inline bool is_so_imm(
unsigned Imm)
const {
182 inline bool is_so_imm_not(
unsigned Imm)
const {
186 inline bool is_t2_so_imm(
unsigned Imm)
const {
190 inline bool is_t2_so_imm_not(
unsigned Imm)
const {
195#include "ARMGenDAGISel.inc"
201 bool tryARMIndexedLoad(
SDNode *
N);
202 bool tryT1IndexedLoad(
SDNode *
N);
203 bool tryT2IndexedLoad(
SDNode *
N);
204 bool tryMVEIndexedLoad(
SDNode *
N);
215 void SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
223 void SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
230 void SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
231 unsigned NumVecs,
const uint16_t *DOpcodes,
235 template <
typename SDValueVector>
236 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
238 template <
typename SDValueVector>
239 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
242 template <
typename SDValueVector>
243 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc);
244 template <
typename SDValueVector>
245 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
EVT InactiveTy);
252 bool HasSaturationOperand);
272 size_t Stride,
size_t TySize);
288 void SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
289 const uint16_t *
const *Opcodes,
bool HasWriteback);
308 void SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
bool isUpdating,
309 unsigned NumVecs,
const uint16_t *DOpcodes,
310 const uint16_t *QOpcodes0 =
nullptr,
311 const uint16_t *QOpcodes1 =
nullptr);
316 bool tryInsertVectorElt(
SDNode *
N);
321 bool tryReadRegister(
SDNode *
N);
322 bool tryWriteRegister(
SDNode *
N);
326 void SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI);
328 void SelectCMP_SWAP(
SDNode *
N);
334 std::vector<SDValue> &OutOps)
override;
356 bool canExtractShiftFromMul(
const SDValue &
N,
unsigned MaxShift,
357 unsigned &PowerOfTwo,
SDValue &NewMulConst)
const;
369 ID,
std::make_unique<ARMDAGToDAGISel>(tm, OptLevel)) {}
373char ARMDAGToDAGISelLegacy::ID = 0;
380 if (
N->getOpcode() ==
ISD::Constant &&
N->getValueType(0) == MVT::i32) {
381 Imm =
N->getAsZExtVal();
397 return N->getOpcode() == Opc &&
406 int RangeMin,
int RangeMax,
407 int &ScaledConstant) {
408 assert(Scale > 0 &&
"Invalid scale!");
415 ScaledConstant = (int)
C->getZExtValue();
416 if ((ScaledConstant % Scale) != 0)
419 ScaledConstant /= Scale;
420 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
423void ARMDAGToDAGISel::PreprocessISelDAG() {
424 if (!Subtarget->hasV6T2Ops())
427 bool isThumb2 = Subtarget->isThumb();
444 unsigned And_imm = 0;
454 if (TZ != 1 && TZ != 2)
466 if (And_imm & (And_imm + 1))
471 unsigned Srl_imm = 0;
482 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
485 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
486 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
493 CurDAG->getConstant(Srl_imm + TZ,
SDLoc(Srl),
497 CurDAG->getConstant(And_imm,
SDLoc(Srl), MVT::i32));
499 N1, CurDAG->getConstant(TZ,
SDLoc(Srl), MVT::i32));
500 CurDAG->UpdateNodeOperands(&
N, N0, N1);
507bool ARMDAGToDAGISel::hasNoVMLxHazardUse(
SDNode *
N)
const {
508 if (OptLevel == CodeGenOptLevel::None)
511 if (!Subtarget->hasVMLxHazards())
520 if (
Use->isMachineOpcode()) {
522 CurDAG->getSubtarget().getInstrInfo());
528 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
543 return TII->isFpMLxInstruction(Opcode);
549bool ARMDAGToDAGISel::isShifterOpProfitable(
const SDValue &Shift,
558 (ShAmt == 2 || (Subtarget->
isSwift() && ShAmt == 1));
561bool ARMDAGToDAGISel::canExtractShiftFromMul(
const SDValue &
N,
563 unsigned &PowerOfTwo,
570 if (!
N.hasOneUse())
return false;
573 if (!MulConst)
return false;
576 if (!MulConst->
hasOneUse())
return false;
578 if (MulConstVal == 0)
return false;
581 PowerOfTwo = MaxShift;
582 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
584 if (PowerOfTwo == 0)
return false;
588 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
589 NewMulConst = CurDAG->getConstant(NewMulConstVal,
SDLoc(
N), MVT::i32);
592 return NewCost < OldCost;
596 CurDAG->RepositionNode(
N.getNode()->getIterator(),
M.getNode());
600bool ARMDAGToDAGISel::SelectImmShifterOperand(
SDValue N,
603 bool CheckProfitability) {
610 unsigned PowerOfTwo = 0;
612 if (canExtractShiftFromMul(
N, 31, PowerOfTwo, NewMulConst)) {
615 replaceDAGValue(
N.getOperand(1), NewMulConst);
617 Opc = CurDAG->getTargetConstant(
629 BaseReg =
N.getOperand(0);
630 unsigned ShImmVal = 0;
632 if (!RHS)
return false;
633 ShImmVal =
RHS->getZExtValue() & 31;
639bool ARMDAGToDAGISel::SelectRegShifterOperand(
SDValue N,
643 bool CheckProfitability) {
653 BaseReg =
N.getOperand(0);
654 unsigned ShImmVal = 0;
656 if (RHS)
return false;
658 ShReg =
N.getOperand(1);
659 if (CheckProfitability && !isShifterOpProfitable(
N, ShOpcVal, ShImmVal))
671 return CurDAG->haveNoCommonBitsSet(
N, Parent->
getOperand(1));
675bool ARMDAGToDAGISel::SelectAddrModeImm12(
SDValue N,
682 !CurDAG->isBaseWithConstantOffset(
N)) {
685 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
686 Base = CurDAG->getTargetFrameIndex(
687 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
688 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
696 Base =
N.getOperand(0);
699 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
704 int RHSC = (int)
RHS->getSExtValue();
708 if (RHSC > -0x1000 && RHSC < 0x1000) {
709 Base =
N.getOperand(0);
711 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
712 Base = CurDAG->getTargetFrameIndex(
713 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
715 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
722 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
734 int RHSC = (int)
RHS->getZExtValue();
743 unsigned ShAmt =
Log2_32(RHSC);
756 !CurDAG->isBaseWithConstantOffset(
N))
763 -0x1000+1, 0x1000, RHSC))
773 Base =
N.getOperand(0);
780 dyn_cast<ConstantSDNode>(
N.getOperand(1).getOperand(1))) {
781 ShAmt = Sh->getZExtValue();
782 if (isShifterOpProfitable(
Offset, ShOpcVal, ShAmt))
783 Offset =
N.getOperand(1).getOperand(0);
796 N.getOperand(0).hasOneUse())) {
802 dyn_cast<ConstantSDNode>(
N.getOperand(0).getOperand(1))) {
803 ShAmt = Sh->getZExtValue();
804 if (isShifterOpProfitable(
N.getOperand(0), ShOpcVal, ShAmt)) {
805 Offset =
N.getOperand(0).getOperand(0);
806 Base =
N.getOperand(1);
820 unsigned PowerOfTwo = 0;
822 if (canExtractShiftFromMul(
Offset, 31, PowerOfTwo, NewMulConst)) {
824 replaceDAGValue(
Offset.getOperand(1), NewMulConst);
825 Offset = Handle.getValue();
838 unsigned Opcode =
Op->getOpcode();
840 ? cast<LoadSDNode>(
Op)->getAddressingMode()
841 : cast<StoreSDNode>(
Op)->getAddressingMode();
855 ShAmt = Sh->getZExtValue();
856 if (isShifterOpProfitable(
N, ShOpcVal, ShAmt))
874 unsigned Opcode =
Op->getOpcode();
876 ? cast<LoadSDNode>(
Op)->getAddressingMode()
877 : cast<StoreSDNode>(
Op)->getAddressingMode();
883 Offset = CurDAG->getRegister(0, MVT::i32);
884 Opc = CurDAG->getTargetConstant(Val,
SDLoc(
Op), MVT::i32);
894 unsigned Opcode =
Op->getOpcode();
896 ? cast<LoadSDNode>(
Op)->getAddressingMode()
897 : cast<StoreSDNode>(
Op)->getAddressingMode();
902 Offset = CurDAG->getRegister(0, MVT::i32);
917bool ARMDAGToDAGISel::SelectAddrMode3(
SDValue N,
922 Base =
N.getOperand(0);
929 if (!CurDAG->isBaseWithConstantOffset(
N)) {
932 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
933 Base = CurDAG->getTargetFrameIndex(
934 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
936 Offset = CurDAG->getRegister(0, MVT::i32);
945 -256 + 1, 256, RHSC)) {
946 Base =
N.getOperand(0);
948 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
949 Base = CurDAG->getTargetFrameIndex(
950 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
952 Offset = CurDAG->getRegister(0, MVT::i32);
964 Base =
N.getOperand(0);
973 unsigned Opcode =
Op->getOpcode();
975 ? cast<LoadSDNode>(
Op)->getAddressingMode()
976 : cast<StoreSDNode>(
Op)->getAddressingMode();
981 Offset = CurDAG->getRegister(0, MVT::i32);
995 if (!CurDAG->isBaseWithConstantOffset(
N)) {
998 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
999 Base = CurDAG->getTargetFrameIndex(
1000 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1005 Base =
N.getOperand(0);
1014 const int Scale =
FP16 ? 2 : 4;
1017 Base =
N.getOperand(0);
1019 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1020 Base = CurDAG->getTargetFrameIndex(
1021 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1052bool ARMDAGToDAGISel::SelectAddrMode5(
SDValue N,
1057bool ARMDAGToDAGISel::SelectAddrMode5FP16(
SDValue N,
1066 unsigned Alignment = 0;
1068 MemSDNode *MemN = cast<MemSDNode>(Parent);
1070 if (isa<LSBaseSDNode>(MemN) ||
1078 if (MMOAlign.
value() >= MemSize && MemSize > 1)
1079 Alignment = MemSize;
1087 Align = CurDAG->getTargetConstant(Alignment,
SDLoc(
N), MVT::i32);
1100 Offset = CurDAG->getRegister(0, MVT::i32);
1105bool ARMDAGToDAGISel::SelectAddrModePC(
SDValue N,
1130 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
1131 return C->getSExtValue() < 0 &&
C->getSExtValue() >= -255;
1138 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N)) {
1146 Base =
N.getOperand(0);
1155 return SelectThumbAddrModeRRSext(
N,
Base,
Offset);
1159ARMDAGToDAGISel::SelectThumbAddrModeImm5S(
SDValue N,
unsigned Scale,
1163 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1167 if (!CurDAG->isBaseWithConstantOffset(
N)) {
1175 Base =
N.getOperand(0);
1180 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1187 Base =
N.getOperand(0);
1188 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1199 return SelectThumbAddrModeImm5S(
N, 4,
Base, OffImm);
1205 return SelectThumbAddrModeImm5S(
N, 2,
Base, OffImm);
1211 return SelectThumbAddrModeImm5S(
N, 1,
Base, OffImm);
1214bool ARMDAGToDAGISel::SelectThumbAddrModeSP(
SDValue N,
1217 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1223 Base = CurDAG->getTargetFrameIndex(
1224 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1225 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1229 if (!CurDAG->isBaseWithConstantOffset(
N))
1236 Base =
N.getOperand(0);
1237 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1248 Base = CurDAG->getTargetFrameIndex(
1249 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1250 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1260template <
unsigned Shift>
1263 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1267 Base =
N.getOperand(0);
1271 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1278 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1288bool ARMDAGToDAGISel::SelectT2AddrModeImm12(
SDValue N,
1294 !CurDAG->isBaseWithConstantOffset(
N)) {
1297 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1298 Base = CurDAG->getTargetFrameIndex(
1299 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1300 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1308 Base =
N.getOperand(0);
1313 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1317 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1318 if (SelectT2AddrModeImm8(
N,
Base, OffImm))
1322 int RHSC = (int)
RHS->getZExtValue();
1326 if (RHSC >= 0 && RHSC < 0x1000) {
1327 Base =
N.getOperand(0);
1329 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1330 Base = CurDAG->getTargetFrameIndex(
1331 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1333 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1340 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1344template <
unsigned Shift>
1347 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1350 Base =
N.getOperand(0);
1352 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1353 Base = CurDAG->getTargetFrameIndex(
1354 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1360 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1367 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1371bool ARMDAGToDAGISel::SelectT2AddrModeImm8(
SDValue N,
1375 !CurDAG->isBaseWithConstantOffset(
N))
1378 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1379 int RHSC = (int)
RHS->getSExtValue();
1383 if ((RHSC >= -255) && (RHSC < 0)) {
1384 Base =
N.getOperand(0);
1386 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1387 Base = CurDAG->getTargetFrameIndex(
1388 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1390 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1400 unsigned Opcode =
Op->getOpcode();
1402 ? cast<LoadSDNode>(
Op)->getAddressingMode()
1403 : cast<StoreSDNode>(
Op)->getAddressingMode();
1407 ? CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32)
1408 : CurDAG->getTargetConstant(-RHSC,
SDLoc(
N), MVT::i32);
1415template <
unsigned Shift>
1418 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1422 Base =
N.getOperand(0);
1424 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1425 Base = CurDAG->getTargetFrameIndex(
1426 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1432 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1439 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1443template <
unsigned Shift>
1446 return SelectT2AddrModeImm7Offset(
Op,
N, OffImm, Shift);
1452 unsigned Opcode =
Op->getOpcode();
1456 AM = cast<LoadSDNode>(
Op)->getAddressingMode();
1459 AM = cast<StoreSDNode>(
Op)->getAddressingMode();
1462 AM = cast<MaskedLoadSDNode>(
Op)->getAddressingMode();
1465 AM = cast<MaskedStoreSDNode>(
Op)->getAddressingMode();
1476 ? CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32)
1477 : CurDAG->getTargetConstant(-RHSC * (1 << Shift),
SDLoc(
N),
1484template <
int Min,
int Max>
1485bool ARMDAGToDAGISel::SelectImmediateInRange(
SDValue N,
SDValue &OffImm) {
1488 OffImm = CurDAG->getTargetConstant(Val,
SDLoc(
N), MVT::i32);
1494bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(
SDValue N,
1498 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N))
1502 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1503 int RHSC = (int)
RHS->getZExtValue();
1504 if (RHSC >= 0 && RHSC < 0x1000)
1506 else if (RHSC < 0 && RHSC >= -255)
1512 Base =
N.getOperand(0);
1513 OffReg =
N.getOperand(1);
1527 ShAmt = Sh->getZExtValue();
1528 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1539 unsigned PowerOfTwo = 0;
1541 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1543 replaceDAGValue(OffReg.
getOperand(1), NewMulConst);
1549 ShImm = CurDAG->getTargetConstant(ShAmt,
SDLoc(
N), MVT::i32);
1559 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1561 if (
N.getOpcode() !=
ISD::ADD || !CurDAG->isBaseWithConstantOffset(
N))
1569 if (RHSC > 1020 || RHSC % 4 != 0)
1572 Base =
N.getOperand(0);
1574 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1575 Base = CurDAG->getTargetFrameIndex(
1576 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1579 OffImm = CurDAG->getTargetConstant(RHSC/4,
SDLoc(
N), MVT::i32);
1590void ARMDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
1592 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {
MemOp});
1595bool ARMDAGToDAGISel::tryARMIndexedLoad(
SDNode *
N) {
1601 EVT LoadedVT =
LD->getMemoryVT();
1604 unsigned Opcode = 0;
1606 if (LoadedVT == MVT::i32 && isPre &&
1607 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1608 Opcode = ARM::LDR_PRE_IMM;
1610 }
else if (LoadedVT == MVT::i32 && !isPre &&
1611 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1612 Opcode = ARM::LDR_POST_IMM;
1614 }
else if (LoadedVT == MVT::i32 &&
1615 SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1616 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1619 }
else if (LoadedVT == MVT::i16 &&
1620 SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1623 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1624 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1625 }
else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1627 if (SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1629 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1633 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1635 Opcode = ARM::LDRB_PRE_IMM;
1636 }
else if (!isPre &&
1637 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1639 Opcode = ARM::LDRB_POST_IMM;
1640 }
else if (SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1642 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1648 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1652 CurDAG->getRegister(0, MVT::i32), Chain };
1653 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1655 transferMemOperands(
N, New);
1656 ReplaceNode(
N, New);
1662 CurDAG->getRegister(0, MVT::i32), Chain };
1663 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1665 transferMemOperands(
N, New);
1666 ReplaceNode(
N, New);
1674bool ARMDAGToDAGISel::tryT1IndexedLoad(
SDNode *
N) {
1676 EVT LoadedVT =
LD->getMemoryVT();
1682 auto *COffs = dyn_cast<ConstantSDNode>(
LD->getOffset());
1683 if (!COffs || COffs->getZExtValue() != 4)
1693 CurDAG->getRegister(0, MVT::i32), Chain };
1694 SDNode *
New = CurDAG->getMachineNode(ARM::tLDR_postidx,
SDLoc(
N), MVT::i32,
1695 MVT::i32, MVT::Other, Ops);
1696 transferMemOperands(
N, New);
1697 ReplaceNode(
N, New);
1701bool ARMDAGToDAGISel::tryT2IndexedLoad(
SDNode *
N) {
1707 EVT LoadedVT =
LD->getMemoryVT();
1711 unsigned Opcode = 0;
1713 if (SelectT2AddrModeImm8Offset(
N,
LD->getOffset(),
Offset)) {
1716 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1720 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1722 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1727 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1729 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1741 CurDAG->getRegister(0, MVT::i32), Chain };
1742 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1744 transferMemOperands(
N, New);
1745 ReplaceNode(
N, New);
1752bool ARMDAGToDAGISel::tryMVEIndexedLoad(
SDNode *
N) {
1754 unsigned Opcode = 0;
1755 bool isSExtLd, isPre;
1765 LoadedVT =
LD->getMemoryVT();
1769 Chain =
LD->getChain();
1772 Alignment =
LD->getAlign();
1776 PredReg = CurDAG->getRegister(0, MVT::i32);
1781 LoadedVT =
LD->getMemoryVT();
1785 Chain =
LD->getChain();
1788 Alignment =
LD->getAlign();
1792 PredReg =
LD->getMask();
1799 bool CanChangeType = Subtarget->
isLittle() && !isa<MaskedLoadSDNode>(
N);
1802 if (Alignment >=
Align(2) && LoadedVT == MVT::v4i16 &&
1803 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1)) {
1805 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1807 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1808 }
else if (LoadedVT == MVT::v8i8 &&
1809 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1811 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1813 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1814 }
else if (LoadedVT == MVT::v4i8 &&
1815 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1817 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1819 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1820 }
else if (Alignment >=
Align(4) &&
1821 (CanChangeType || LoadedVT == MVT::v4i32 ||
1822 LoadedVT == MVT::v4f32) &&
1823 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 2))
1824 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1825 else if (Alignment >=
Align(2) &&
1826 (CanChangeType || LoadedVT == MVT::v8i16 ||
1827 LoadedVT == MVT::v8f16) &&
1828 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1))
1829 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1830 else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
1831 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0))
1832 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1838 CurDAG->getTargetConstant(Pred,
SDLoc(
N), MVT::i32),
1840 CurDAG->getRegister(0, MVT::i32),
1843 N->getValueType(0), MVT::Other, Ops);
1844 transferMemOperands(
N, New);
1848 CurDAG->RemoveDeadNode(
N);
1856 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1857 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1858 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1859 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1860 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1867 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1868 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1869 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1870 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1871 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1877 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1879 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1880 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1881 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1882 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1888 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1890 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1891 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1892 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1893 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1901 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1902 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1903 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1904 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1905 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1906 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1907 V2, SubReg2, V3, SubReg3 };
1908 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1915 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1917 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1918 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1919 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1920 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1921 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1922 V2, SubReg2, V3, SubReg3 };
1923 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1930 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1932 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1933 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1934 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1935 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1936 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1937 V2, SubReg2, V3, SubReg3 };
1938 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1945 unsigned NumVecs,
bool is64BitVector) {
1946 unsigned NumRegs = NumVecs;
1947 if (!is64BitVector && NumVecs < 3)
1950 unsigned Alignment =
Align->getAsZExtVal();
1951 if (Alignment >= 32 && NumRegs == 4)
1953 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1955 else if (Alignment >= 8)
1960 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1966 default:
return false;
1967 case ARM::VLD1d8wb_fixed :
return true;
1968 case ARM::VLD1d16wb_fixed :
return true;
1969 case ARM::VLD1d64Qwb_fixed :
return true;
1970 case ARM::VLD1d32wb_fixed :
return true;
1971 case ARM::VLD1d64wb_fixed :
return true;
1972 case ARM::VLD1d8TPseudoWB_fixed :
return true;
1973 case ARM::VLD1d16TPseudoWB_fixed :
return true;
1974 case ARM::VLD1d32TPseudoWB_fixed :
return true;
1975 case ARM::VLD1d64TPseudoWB_fixed :
return true;
1976 case ARM::VLD1d8QPseudoWB_fixed :
return true;
1977 case ARM::VLD1d16QPseudoWB_fixed :
return true;
1978 case ARM::VLD1d32QPseudoWB_fixed :
return true;
1979 case ARM::VLD1d64QPseudoWB_fixed :
return true;
1980 case ARM::VLD1q8wb_fixed :
return true;
1981 case ARM::VLD1q16wb_fixed :
return true;
1982 case ARM::VLD1q32wb_fixed :
return true;
1983 case ARM::VLD1q64wb_fixed :
return true;
1984 case ARM::VLD1DUPd8wb_fixed :
return true;
1985 case ARM::VLD1DUPd16wb_fixed :
return true;
1986 case ARM::VLD1DUPd32wb_fixed :
return true;
1987 case ARM::VLD1DUPq8wb_fixed :
return true;
1988 case ARM::VLD1DUPq16wb_fixed :
return true;
1989 case ARM::VLD1DUPq32wb_fixed :
return true;
1990 case ARM::VLD2d8wb_fixed :
return true;
1991 case ARM::VLD2d16wb_fixed :
return true;
1992 case ARM::VLD2d32wb_fixed :
return true;
1993 case ARM::VLD2q8PseudoWB_fixed :
return true;
1994 case ARM::VLD2q16PseudoWB_fixed :
return true;
1995 case ARM::VLD2q32PseudoWB_fixed :
return true;
1996 case ARM::VLD2DUPd8wb_fixed :
return true;
1997 case ARM::VLD2DUPd16wb_fixed :
return true;
1998 case ARM::VLD2DUPd32wb_fixed :
return true;
1999 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return true;
2000 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return true;
2001 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return true;
2008 default:
return false;
2009 case ARM::VST1d8wb_fixed :
return true;
2010 case ARM::VST1d16wb_fixed :
return true;
2011 case ARM::VST1d32wb_fixed :
return true;
2012 case ARM::VST1d64wb_fixed :
return true;
2013 case ARM::VST1q8wb_fixed :
return true;
2014 case ARM::VST1q16wb_fixed :
return true;
2015 case ARM::VST1q32wb_fixed :
return true;
2016 case ARM::VST1q64wb_fixed :
return true;
2017 case ARM::VST1d8TPseudoWB_fixed :
return true;
2018 case ARM::VST1d16TPseudoWB_fixed :
return true;
2019 case ARM::VST1d32TPseudoWB_fixed :
return true;
2020 case ARM::VST1d64TPseudoWB_fixed :
return true;
2021 case ARM::VST1d8QPseudoWB_fixed :
return true;
2022 case ARM::VST1d16QPseudoWB_fixed :
return true;
2023 case ARM::VST1d32QPseudoWB_fixed :
return true;
2024 case ARM::VST1d64QPseudoWB_fixed :
return true;
2025 case ARM::VST2d8wb_fixed :
return true;
2026 case ARM::VST2d16wb_fixed :
return true;
2027 case ARM::VST2d32wb_fixed :
return true;
2028 case ARM::VST2q8PseudoWB_fixed :
return true;
2029 case ARM::VST2q16PseudoWB_fixed :
return true;
2030 case ARM::VST2q32PseudoWB_fixed :
return true;
2038 &&
"Incorrect fixed stride updating instruction.");
2041 case ARM::VLD1d8wb_fixed:
return ARM::VLD1d8wb_register;
2042 case ARM::VLD1d16wb_fixed:
return ARM::VLD1d16wb_register;
2043 case ARM::VLD1d32wb_fixed:
return ARM::VLD1d32wb_register;
2044 case ARM::VLD1d64wb_fixed:
return ARM::VLD1d64wb_register;
2045 case ARM::VLD1q8wb_fixed:
return ARM::VLD1q8wb_register;
2046 case ARM::VLD1q16wb_fixed:
return ARM::VLD1q16wb_register;
2047 case ARM::VLD1q32wb_fixed:
return ARM::VLD1q32wb_register;
2048 case ARM::VLD1q64wb_fixed:
return ARM::VLD1q64wb_register;
2049 case ARM::VLD1d64Twb_fixed:
return ARM::VLD1d64Twb_register;
2050 case ARM::VLD1d64Qwb_fixed:
return ARM::VLD1d64Qwb_register;
2051 case ARM::VLD1d8TPseudoWB_fixed:
return ARM::VLD1d8TPseudoWB_register;
2052 case ARM::VLD1d16TPseudoWB_fixed:
return ARM::VLD1d16TPseudoWB_register;
2053 case ARM::VLD1d32TPseudoWB_fixed:
return ARM::VLD1d32TPseudoWB_register;
2054 case ARM::VLD1d64TPseudoWB_fixed:
return ARM::VLD1d64TPseudoWB_register;
2055 case ARM::VLD1d8QPseudoWB_fixed:
return ARM::VLD1d8QPseudoWB_register;
2056 case ARM::VLD1d16QPseudoWB_fixed:
return ARM::VLD1d16QPseudoWB_register;
2057 case ARM::VLD1d32QPseudoWB_fixed:
return ARM::VLD1d32QPseudoWB_register;
2058 case ARM::VLD1d64QPseudoWB_fixed:
return ARM::VLD1d64QPseudoWB_register;
2059 case ARM::VLD1DUPd8wb_fixed :
return ARM::VLD1DUPd8wb_register;
2060 case ARM::VLD1DUPd16wb_fixed :
return ARM::VLD1DUPd16wb_register;
2061 case ARM::VLD1DUPd32wb_fixed :
return ARM::VLD1DUPd32wb_register;
2062 case ARM::VLD1DUPq8wb_fixed :
return ARM::VLD1DUPq8wb_register;
2063 case ARM::VLD1DUPq16wb_fixed :
return ARM::VLD1DUPq16wb_register;
2064 case ARM::VLD1DUPq32wb_fixed :
return ARM::VLD1DUPq32wb_register;
2065 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return ARM::VLD2DUPq8OddPseudoWB_register;
2066 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return ARM::VLD2DUPq16OddPseudoWB_register;
2067 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return ARM::VLD2DUPq32OddPseudoWB_register;
2069 case ARM::VST1d8wb_fixed:
return ARM::VST1d8wb_register;
2070 case ARM::VST1d16wb_fixed:
return ARM::VST1d16wb_register;
2071 case ARM::VST1d32wb_fixed:
return ARM::VST1d32wb_register;
2072 case ARM::VST1d64wb_fixed:
return ARM::VST1d64wb_register;
2073 case ARM::VST1q8wb_fixed:
return ARM::VST1q8wb_register;
2074 case ARM::VST1q16wb_fixed:
return ARM::VST1q16wb_register;
2075 case ARM::VST1q32wb_fixed:
return ARM::VST1q32wb_register;
2076 case ARM::VST1q64wb_fixed:
return ARM::VST1q64wb_register;
2077 case ARM::VST1d8TPseudoWB_fixed:
return ARM::VST1d8TPseudoWB_register;
2078 case ARM::VST1d16TPseudoWB_fixed:
return ARM::VST1d16TPseudoWB_register;
2079 case ARM::VST1d32TPseudoWB_fixed:
return ARM::VST1d32TPseudoWB_register;
2080 case ARM::VST1d64TPseudoWB_fixed:
return ARM::VST1d64TPseudoWB_register;
2081 case ARM::VST1d8QPseudoWB_fixed:
return ARM::VST1d8QPseudoWB_register;
2082 case ARM::VST1d16QPseudoWB_fixed:
return ARM::VST1d16QPseudoWB_register;
2083 case ARM::VST1d32QPseudoWB_fixed:
return ARM::VST1d32QPseudoWB_register;
2084 case ARM::VST1d64QPseudoWB_fixed:
return ARM::VST1d64QPseudoWB_register;
2086 case ARM::VLD2d8wb_fixed:
return ARM::VLD2d8wb_register;
2087 case ARM::VLD2d16wb_fixed:
return ARM::VLD2d16wb_register;
2088 case ARM::VLD2d32wb_fixed:
return ARM::VLD2d32wb_register;
2089 case ARM::VLD2q8PseudoWB_fixed:
return ARM::VLD2q8PseudoWB_register;
2090 case ARM::VLD2q16PseudoWB_fixed:
return ARM::VLD2q16PseudoWB_register;
2091 case ARM::VLD2q32PseudoWB_fixed:
return ARM::VLD2q32PseudoWB_register;
2093 case ARM::VST2d8wb_fixed:
return ARM::VST2d8wb_register;
2094 case ARM::VST2d16wb_fixed:
return ARM::VST2d16wb_register;
2095 case ARM::VST2d32wb_fixed:
return ARM::VST2d32wb_register;
2096 case ARM::VST2q8PseudoWB_fixed:
return ARM::VST2q8PseudoWB_register;
2097 case ARM::VST2q16PseudoWB_fixed:
return ARM::VST2q16PseudoWB_register;
2098 case ARM::VST2q32PseudoWB_fixed:
return ARM::VST2q32PseudoWB_register;
2100 case ARM::VLD2DUPd8wb_fixed:
return ARM::VLD2DUPd8wb_register;
2101 case ARM::VLD2DUPd16wb_fixed:
return ARM::VLD2DUPd16wb_register;
2102 case ARM::VLD2DUPd32wb_fixed:
return ARM::VLD2DUPd32wb_register;
2111 auto C = dyn_cast<ConstantSDNode>(Inc);
2115void ARMDAGToDAGISel::SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2119 assert(Subtarget->hasNEON());
2120 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLD NumVecs out-of-range");
2124 bool IsIntrinsic = !isUpdating;
2126 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2127 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2131 EVT VT =
N->getValueType(0);
2133 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2161 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2166 std::vector<EVT> ResTys;
2167 ResTys.push_back(ResTy);
2169 ResTys.push_back(MVT::i32);
2170 ResTys.push_back(MVT::Other);
2173 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2178 if (is64BitVector || NumVecs <= 2) {
2179 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2184 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2200 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2210 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2211 const SDValue OpsA[] = { MemAddr,
Align, Reg0, ImplDef, Pred, Reg0, Chain };
2213 ResTy, AddrTy, MVT::Other, OpsA);
2220 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2222 "only constant post-increment update allowed for VLD3/4");
2230 VLd = CurDAG->getMachineNode(QOpcodes1[
OpcodeIndex], dl, ResTys, Ops);
2235 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {
MemOp});
2238 ReplaceNode(
N, VLd);
2244 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2245 ARM::qsub_3 == ARM::qsub_0 + 3,
2246 "Unexpected subreg numbering");
2247 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2248 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2250 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2254 CurDAG->RemoveDeadNode(
N);
2257void ARMDAGToDAGISel::SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2261 assert(Subtarget->hasNEON());
2262 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VST NumVecs out-of-range");
2266 bool IsIntrinsic = !isUpdating;
2268 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2269 unsigned Vec0Idx = 3;
2270 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2276 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2278 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2302 std::vector<EVT> ResTys;
2304 ResTys.push_back(MVT::i32);
2305 ResTys.push_back(MVT::Other);
2308 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2312 if (is64BitVector || NumVecs <= 2) {
2315 SrcReg =
N->getOperand(Vec0Idx);
2316 }
else if (is64BitVector) {
2318 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2319 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2321 SrcReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2327 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2328 :
N->getOperand(Vec0Idx + 3);
2329 SrcReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2333 SDValue Q0 =
N->getOperand(Vec0Idx);
2334 SDValue Q1 =
N->getOperand(Vec0Idx + 1);
2335 SrcReg =
SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2338 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2343 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2361 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2364 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {
MemOp});
2366 ReplaceNode(
N, VSt);
2374 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2375 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2378 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2379 :
N->getOperand(Vec0Idx + 3);
2380 SDValue RegSeq =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2384 const SDValue OpsA[] = { MemAddr,
Align, Reg0, RegSeq, Pred, Reg0, Chain };
2388 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {
MemOp});
2395 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2397 "only constant post-increment update allowed for VST3/4");
2407 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {
MemOp});
2408 ReplaceNode(
N, VStB);
2411void ARMDAGToDAGISel::SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
2415 assert(Subtarget->hasNEON());
2416 assert(NumVecs >=2 && NumVecs <= 4 &&
"VLDSTLane NumVecs out-of-range");
2420 bool IsIntrinsic = !isUpdating;
2422 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2423 unsigned Vec0Idx = 3;
2424 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2430 unsigned Lane =
N->getConstantOperandVal(Vec0Idx + NumVecs);
2431 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2434 unsigned Alignment = 0;
2436 Alignment =
Align->getAsZExtVal();
2438 if (Alignment > NumBytes)
2439 Alignment = NumBytes;
2440 if (Alignment < 8 && Alignment < NumBytes)
2443 Alignment = (Alignment & -Alignment);
2447 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2467 std::vector<EVT> ResTys;
2469 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2473 MVT::i64, ResTyElts));
2476 ResTys.push_back(MVT::i32);
2477 ResTys.push_back(MVT::Other);
2480 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2486 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2489 Ops.
push_back(IsImmUpdate ? Reg0 : Inc);
2493 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2494 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2497 SuperReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2499 SuperReg =
SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2503 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2504 :
N->getOperand(Vec0Idx + 3);
2506 SuperReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2508 SuperReg =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2516 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2518 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2519 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {
MemOp});
2521 ReplaceNode(
N, VLdLn);
2527 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2528 ARM::qsub_3 == ARM::qsub_0 + 3,
2529 "Unexpected subreg numbering");
2530 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2531 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2533 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2537 CurDAG->RemoveDeadNode(
N);
2540template <
typename SDValueVector>
2541void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2543 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2544 Ops.push_back(PredicateMask);
2545 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2548template <
typename SDValueVector>
2549void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2552 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2553 Ops.push_back(PredicateMask);
2554 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2555 Ops.push_back(Inactive);
2558template <
typename SDValueVector>
2559void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc) {
2560 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2561 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2562 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2565template <
typename SDValueVector>
2566void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2568 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2569 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2570 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2572 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2581 switch (
N->getValueType(1).getVectorElementType().getSizeInBits()) {
2583 Opcode = Opcodes[0];
2586 Opcode = Opcodes[1];
2594 int32_t ImmValue =
N->getConstantOperandVal(3);
2595 Ops.
push_back(getI32Imm(ImmValue, Loc));
2598 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2600 AddEmptyMVEPredicateToOps(Ops, Loc);
2613 transferMemOperands(
N, New);
2614 CurDAG->RemoveDeadNode(
N);
2619 bool HasSaturationOperand) {
2629 int32_t ImmValue =
N->getConstantOperandVal(3);
2630 Ops.
push_back(getI32Imm(ImmValue, Loc));
2636 if (HasSaturationOperand) {
2637 int32_t SatOp =
N->getConstantOperandVal(4);
2638 int SatBit = (SatOp == 64 ? 0 : 1);
2645 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
2647 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2650void ARMDAGToDAGISel::SelectMVE_VADCSBC(
SDNode *
N,
uint16_t OpcodeWithCarry,
2661 Ops.
push_back(
N->getOperand(FirstInputOp + 1));
2662 SDValue CarryIn =
N->getOperand(FirstInputOp + 2);
2663 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2666 if (CarryInConstant &&
2667 (CarryInConstant->
getZExtValue() & CarryMask) == CarryExpected) {
2668 Opcode = OpcodeWithNoCarry;
2671 Opcode = OpcodeWithCarry;
2675 AddMVEPredicateToOps(Ops, Loc,
2676 N->getOperand(FirstInputOp + 3),
2677 N->getOperand(FirstInputOp - 1));
2679 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2681 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2692 int32_t ImmValue =
N->getConstantOperandVal(3);
2693 Ops.
push_back(getI32Imm(ImmValue, Loc));
2696 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2698 AddEmptyMVEPredicateToOps(Ops, Loc);
2700 CurDAG->SelectNodeTo(
N, ARM::MVE_VSHLC,
N->getVTList(),
ArrayRef(Ops));
2704 assert(isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant");
2714 size_t Stride,
size_t TySize) {
2715 assert(TySize < Stride &&
"Invalid TySize");
2721 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2723 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2726 auto OpIsZero = [
N](
size_t OpNo) {
2732 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2734 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2736 Opcodes += 4 * Stride;
2738 Opcodes += 2 * Stride;
2755 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(8));
2757 AddEmptyMVEPredicateToOps(Ops, Loc);
2759 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2765 EVT VecTy =
N->getOperand(6).getValueType();
2778 SelectBaseMVE_VMLLDAV(
N,
Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2785 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2787 "bad vector element size");
2788 SelectBaseMVE_VMLLDAV(
N,
Predicated, OpcodesS, OpcodesU, 1, 0);
2791void ARMDAGToDAGISel::SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
2793 bool HasWriteback) {
2794 EVT VT =
N->getValueType(0);
2800 OurOpcodes = Opcodes[0];
2803 OurOpcodes = Opcodes[1];
2806 OurOpcodes = Opcodes[2];
2814 unsigned PtrOperand = HasWriteback ? 1 : 2;
2817 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2820 for (
unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2821 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2823 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2830 ResultTys = {DataTy, MVT::i32, MVT::Other};
2831 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2833 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2837 for (i = 0; i < NumVecs; i++)
2839 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2844 CurDAG->RemoveDeadNode(
N);
2847void ARMDAGToDAGISel::SelectMVE_VxDUP(
SDNode *
N,
const uint16_t *Opcodes,
2849 EVT VT =
N->getValueType(0);
2855 Opcode = Opcodes[0];
2858 Opcode = Opcodes[1];
2861 Opcode = Opcodes[2];
2872 Inactive =
N->getOperand(OpIdx++);
2878 SDValue ImmOp =
N->getOperand(OpIdx++);
2880 Ops.
push_back(getI32Imm(ImmValue, Loc));
2883 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(OpIdx), Inactive);
2885 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2887 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2891 size_t NumExtraOps,
bool HasAccum) {
2892 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2899 SDValue ImmCorpoc =
N->getOperand(OpIdx++);
2901 Ops.
push_back(getI32Imm(ImmCoprocVal, Loc));
2906 SDValue AccLo =
N->getOperand(OpIdx++);
2907 SDValue AccHi =
N->getOperand(OpIdx++);
2914 for (
size_t I = 0;
I < NumExtraOps;
I++)
2925 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2931 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops);
2938 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2942 for (
size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2943 if (
SDValue(
N, ResIdx).use_empty())
2945 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2946 MVT::i32, ResultPair);
2950 CurDAG->RemoveDeadNode(
N);
2953void ARMDAGToDAGISel::SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
2954 bool isUpdating,
unsigned NumVecs,
2958 assert(Subtarget->hasNEON());
2959 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLDDup NumVecs out-of-range");
2963 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2964 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2968 EVT VT =
N->getValueType(0);
2971 unsigned Alignment = 0;
2973 Alignment =
Align->getAsZExtVal();
2975 if (Alignment > NumBytes)
2976 Alignment = NumBytes;
2977 if (Alignment < 8 && Alignment < NumBytes)
2980 Alignment = (Alignment & -Alignment);
2984 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
3006 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
3011 std::vector<EVT> ResTys;
3012 ResTys.push_back(ResTy);
3014 ResTys.push_back(MVT::i32);
3015 ResTys.push_back(MVT::Other);
3018 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3023 unsigned Opc = is64BitVector ? DOpcodes[
OpcodeIndex]
3039 if (is64BitVector || NumVecs == 1) {
3043 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3044 const SDValue OpsA[] = {MemAddr,
Align, ImplDef, Pred, Reg0, Chain};
3055 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3059 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {
MemOp});
3066 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7,
"Unexpected subreg numbering");
3067 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3068 for (
unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3070 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3076 CurDAG->RemoveDeadNode(
N);
3079bool ARMDAGToDAGISel::tryInsertVectorElt(
SDNode *
N) {
3080 if (!Subtarget->hasMVEIntegerOps())
3094 (VT != MVT::v8f16 && VT != MVT::v8i16) || (Ins2.
getValueType() != VT))
3099 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3126 ExtractLane1 == ExtractLane2 + 1) {
3127 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3128 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val1.
getOperand(0));
3129 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3130 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.
getOperand(0),
3132 ReplaceUses(Ins1, NewIns);
3138 if (VT == MVT::v8i16 && Subtarget->hasFullFP16()) {
3139 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3140 ARM::ssub_0 + ExtractLane1 / 2, dl, MVT::f32, Val1.
getOperand(0));
3141 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3142 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val2.
getOperand(0));
3143 if (ExtractLane1 % 2 != 0)
3144 Inp1 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0);
3145 if (ExtractLane2 % 2 != 0)
3146 Inp2 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0);
3147 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1);
3149 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3151 ReplaceUses(Ins1, NewIns);
3158 if (VT == MVT::v8f16 && Subtarget->hasFullFP16()) {
3159 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1);
3161 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3163 ReplaceUses(Ins1, NewIns);
3170bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(
SDNode *
N,
3173 bool FixedToFloat) {
3174 auto Type =
N->getValueType(0);
3176 if (ScalarBits > 32)
3182 if (ScalarBits == 16 && !FMulFlags.
hasNoInfs() && IsUnsigned)
3207 if (!isa<ConstantSDNode>(ImmNode.
getOperand(0)))
3213 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3214 APInt(ScalarBits, Imm));
3231 if (!ImmAPF.getExactInverse(&ToConvert))
3234 APSInt Converted(64,
false);
3238 if (!IsExact || !Converted.isPowerOf2())
3241 unsigned FracBits = Converted.logBase2();
3242 if (FracBits > ScalarBits)
3246 VecVal, CurDAG->getConstant(FracBits,
SDLoc(
N), MVT::i32)};
3247 AddEmptyMVEPredicateToOps(Ops,
SDLoc(
N),
Type);
3249 unsigned int Opcode;
3250 switch (ScalarBits) {
3253 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3255 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3259 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3261 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3268 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
SDLoc(
N),
Type, Ops));
3272bool ARMDAGToDAGISel::tryFP_TO_INT(
SDNode *
N,
SDLoc dl) {
3274 if (!Subtarget->hasMVEFloatOps())
3277 if (!
Type.isVector())
3288 if (
Node->getOperand(0) !=
Node->getOperand(1))
3293 if (ScalarBits == 16 && !
Flags.hasNoInfs() && IsUnsigned)
3297 switch (ScalarBits) {
3299 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3302 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3306 CurDAG->getConstant(1, dl, MVT::i32)};
3307 AddEmptyMVEPredicateToOps(Ops, dl,
Type);
3309 ReplaceNode(
N, CurDAG->getMachineNode(Opcode, dl,
Type, Ops));
3316 return transformFixedFloatingPointConversion(
N,
Node, IsUnsigned,
false);
3319bool ARMDAGToDAGISel::tryFMULFixed(
SDNode *
N,
SDLoc dl) {
3321 if (!Subtarget->hasMVEFloatOps())
3323 auto Type =
N->getValueType(0);
3324 if (!
Type.isVector())
3327 auto LHS =
N->getOperand(0);
3331 return transformFixedFloatingPointConversion(
3335bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned) {
3336 if (!Subtarget->hasV6T2Ops())
3340 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3341 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3345 unsigned And_imm = 0;
3350 if (And_imm & (And_imm + 1))
3353 unsigned Srl_imm = 0;
3356 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3361 And_imm &= -1U >> Srl_imm;
3365 unsigned LSB = Srl_imm;
3367 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3369 if ((LSB + Width + 1) ==
N->getValueType(0).getSizeInBits()) {
3371 if (Subtarget->isThumb()) {
3372 Opc =
isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3373 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3374 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3375 getAL(CurDAG, dl), Reg0, Reg0 };
3376 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3385 SDValue Ops[] = {
N->getOperand(0).getOperand(0), ShOpc,
3386 getAL(CurDAG, dl), Reg0, Reg0 };
3387 CurDAG->SelectNodeTo(
N, ARM::MOVsi, MVT::i32, Ops);
3391 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3392 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3393 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3394 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3395 getAL(CurDAG, dl), Reg0 };
3396 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3404 unsigned Shl_imm = 0;
3406 assert(Shl_imm > 0 && Shl_imm < 32 &&
"bad amount in shift node!");
3407 unsigned Srl_imm = 0;
3409 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3411 unsigned Width = 32 - Srl_imm - 1;
3412 int LSB = Srl_imm - Shl_imm;
3415 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3416 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3417 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3418 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3419 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3420 getAL(CurDAG, dl), Reg0 };
3421 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3429 unsigned Srl_imm = 0;
3433 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3436 unsigned Width = MSB - LSB;
3437 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3438 assert(Srl_imm + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3439 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3440 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
3441 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3442 getAL(CurDAG, dl), Reg0 };
3443 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3449 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3455 if (LSB + Width > 32)
3458 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3459 assert(LSB + Width <= 32 &&
"Shouldn't create an invalid ubfx");
3460 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3461 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3462 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
3463 getAL(CurDAG, dl), Reg0 };
3464 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3481bool ARMDAGToDAGISel::tryABSOp(
SDNode *
N){
3482 SDValue SUBSrc0 =
N->getOperand(0);
3483 SDValue SUBSrc1 =
N->getOperand(1);
3484 EVT VT =
N->getValueType(0);
3500 if (XORSrc1 == SUBSrc1 && XORSrc0 == SRASrc0 && XType.
isInteger() &&
3502 unsigned Opcode = Subtarget->
isThumb2() ? ARM::t2ABS : ARM::ABS;
3503 CurDAG->SelectNodeTo(
N, Opcode, VT, XORSrc0);
3511void ARMDAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3513 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3514 if (MemTy == MVT::i8)
3515 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3516 else if (MemTy == MVT::i16)
3517 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3518 else if (MemTy == MVT::i32)
3519 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32;
3523 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3525 SDNode *CmpSwap = CurDAG->getMachineNode(
3527 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
3530 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3534 CurDAG->RemoveDeadNode(
N);
3537static std::optional<std::pair<unsigned, unsigned>>
3539 unsigned FirstOne =
A.getBitWidth() -
A.countl_zero() - 1;
3540 unsigned LastOne =
A.countr_zero();
3541 if (
A.popcount() != (FirstOne - LastOne + 1))
3542 return std::nullopt;
3543 return std::make_pair(FirstOne, LastOne);
3546void ARMDAGToDAGISel::SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI) {
3548 SwitchEQNEToPLMI =
false;
3550 if (!Subtarget->isThumb())
3557 if (!
And->hasOneUse())
3564 auto C = dyn_cast<ConstantSDNode>(
And.getOperand(1));
3578 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3579 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3580 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3581 CurDAG->getRegister(0, MVT::i32) };
3582 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3584 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
3585 CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3586 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3587 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3591 if (
Range->second == 0) {
3593 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3594 ReplaceNode(
And.getNode(), NewN);
3595 }
else if (
Range->first == 31) {
3597 NewN = EmitShift(ARM::tLSRri,
X,
Range->second);
3598 ReplaceNode(
And.getNode(), NewN);
3602 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3603 ReplaceNode(
And.getNode(), NewN);
3605 SwitchEQNEToPLMI =
true;
3606 }
else if (!Subtarget->hasV6T2Ops()) {
3609 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3610 NewN = EmitShift(ARM::tLSRri,
SDValue(NewN, 0),
3612 ReplaceNode(
And.getNode(), NewN);
3617 unsigned Opc128[3]) {
3619 "Unexpected vector shuffle length");
3632void ARMDAGToDAGISel::Select(
SDNode *
N) {
3635 if (
N->isMachineOpcode()) {
3640 switch (
N->getOpcode()) {
3662 cast<RegisterSDNode>(
Ptr.getOperand(1))->getReg() == ARM::SP &&
3663 Ptr.getOperand(0) ==
ST->getChain()) {
3665 CurDAG->getRegister(ARM::SP, MVT::i32),
3666 CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
3668 CurDAG->getRegister(0, MVT::i32),
3671 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
3673 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3674 ReplaceNode(
N, ResNode);
3681 if (tryWriteRegister(
N))
3685 if (tryReadRegister(
N))
3690 if (tryInlineAsm(
N))
3700 unsigned Val =
N->getAsZExtVal();
3703 !Subtarget->genExecuteOnly()) {
3704 SDValue CPIdx = CurDAG->getTargetConstantPool(
3706 TLI->getPointerTy(CurDAG->getDataLayout()));
3709 if (Subtarget->isThumb()) {
3713 CurDAG->getRegister(0, MVT::i32),
3714 CurDAG->getEntryNode()
3716 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
3721 CurDAG->getTargetConstant(0, dl, MVT::i32),
3723 CurDAG->getRegister(0, MVT::i32),
3724 CurDAG->getEntryNode()
3726 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
3737 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3739 ReplaceNode(
N, ResNode);
3748 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
3749 SDValue TFI = CurDAG->getTargetFrameIndex(
3750 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3757 CurDAG->SelectNodeTo(
N, ARM::tADDframe, MVT::i32, TFI,
3758 CurDAG->getTargetConstant(0, dl, MVT::i32));
3761 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3762 ARM::t2ADDri : ARM::ADDri);
3763 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3764 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3765 CurDAG->getRegister(0, MVT::i32) };
3766 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3771 if (tryInsertVectorElt(
N))
3776 if (tryV6T2BitfieldExtractOp(
N,
false))
3781 if (tryV6T2BitfieldExtractOp(
N,
true))
3788 if (tryFP_TO_INT(
N, dl))
3792 if (tryFMULFixed(
N, dl))
3799 unsigned RHSV =
C->getZExtValue();
3802 unsigned ShImm =
Log2_32(RHSV-1);
3807 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3808 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3809 if (Subtarget->isThumb()) {
3810 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3811 CurDAG->SelectNodeTo(
N, ARM::t2ADDrs, MVT::i32, Ops);
3816 CurDAG->SelectNodeTo(
N, ARM::ADDrsi, MVT::i32, Ops);
3821 unsigned ShImm =
Log2_32(RHSV+1);
3826 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3827 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3828 if (Subtarget->isThumb()) {
3829 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3830 CurDAG->SelectNodeTo(
N, ARM::t2RSBrs, MVT::i32, Ops);
3835 CurDAG->SelectNodeTo(
N, ARM::RSBrsi, MVT::i32, Ops);
3843 if (tryV6T2BitfieldExtractOp(
N,
false))
3849 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3850 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3857 bool PreferImmediateEncoding =
3858 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3859 if (!PreferImmediateEncoding &&
3865 CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
3869 CurDAG->RepositionNode(
N->getIterator(), NewImm.
getNode());
3871 if (!Subtarget->hasThumb2()) {
3872 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3873 N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3874 CurDAG->getRegister(0, MVT::i32)};
3875 ReplaceNode(
N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
3878 SDValue Ops[] = {
N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3879 CurDAG->getRegister(0, MVT::i32),
3880 CurDAG->getRegister(0, MVT::i32)};
3882 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
3893 EVT VT =
N->getValueType(0);
3896 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3898 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3901 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
3902 N1C = dyn_cast<ConstantSDNode>(N1);
3910 unsigned N1CVal = N1C->getZExtValue();
3912 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3913 (N1CVal & 0xffffU) == 0xffffU &&
3914 (N2CVal & 0xffffU) == 0x0U) {
3915 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3918 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
3919 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3927 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3928 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
3929 N->getOperand(2),
N->getOperand(3),
3931 CurDAG->getRegister(0, MVT::i32) };
3932 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
3936 if (Subtarget->isThumb()) {
3937 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3938 N->getOperand(3),
getAL(CurDAG, dl),
3939 CurDAG->getRegister(0, MVT::i32)};
3941 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
3944 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3945 N->getOperand(3),
getAL(CurDAG, dl),
3946 CurDAG->getRegister(0, MVT::i32),
3947 CurDAG->getRegister(0, MVT::i32) };
3948 ReplaceNode(
N, CurDAG->getMachineNode(
3949 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3950 MVT::i32, MVT::i32, Ops));
3955 if (Subtarget->isThumb()) {
3956 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3957 N->getOperand(3),
getAL(CurDAG, dl),
3958 CurDAG->getRegister(0, MVT::i32)};
3960 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
3963 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3964 N->getOperand(3),
getAL(CurDAG, dl),
3965 CurDAG->getRegister(0, MVT::i32),
3966 CurDAG->getRegister(0, MVT::i32) };
3967 ReplaceNode(
N, CurDAG->getMachineNode(
3968 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3969 MVT::i32, MVT::i32, Ops));
3974 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3983 if (Subtarget->isThumb())
3984 assert(Subtarget->hasThumb2() &&
3985 "This pattern should not be generated for Thumb");
3987 SDValue SmulLoHi =
N->getOperand(1);
3992 N->getOperand(1) != SmulLoHi.
getValue(1) ||
3996 unsigned Opc = Subtarget->
isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3998 N->getOperand(0),
getAL(CurDAG, dl),
3999 CurDAG->getRegister(0, MVT::i32) };
4000 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
4004 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4006 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
4007 if (tryT2IndexedLoad(
N))
4009 }
else if (Subtarget->isThumb()) {
4010 if (tryT1IndexedLoad(
N))
4012 }
else if (tryARMIndexedLoad(
N))
4018 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4023 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopSetup, dl, MVT::i32,
4025 ReplaceUses(
N, New);
4026 CurDAG->RemoveDeadNode(
N);
4030 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopStart, dl, MVT::Other,
4031 N->getOperand(1),
N->getOperand(2),
4033 ReplaceUses(
N, New);
4034 CurDAG->RemoveDeadNode(
N);
4038 SDValue Ops[] = {
N->getOperand(1),
4041 unsigned Opc = ARM::t2LoopEnd;
4042 SDNode *
New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
4043 ReplaceUses(
N, New);
4044 CurDAG->RemoveDeadNode(
N);
4051 const SDValue &Chain =
N->getOperand(0);
4053 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4054 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4060 RegOffset = CurDAG->getRegister(0, MVT::i32);
4062 SDValue Ops[] = {
Base, RegOffset, ImmOffset, Chain};
4063 SDNode *
New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4064 {MVT::Untyped, MVT::Other}, Ops);
4065 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4067 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4069 transferMemOperands(
N, New);
4073 CurDAG->RemoveDeadNode(
N);
4080 const SDValue &Chain =
N->getOperand(0);
4082 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4083 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4089 RegOffset = CurDAG->getRegister(0, MVT::i32);
4094 SDNode *
New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops);
4095 transferMemOperands(
N, New);
4097 CurDAG->RemoveDeadNode(
N);
4101 SDValue Ops[] = {
N->getOperand(1),
4105 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4106 CurDAG->getVTList(MVT::i32, MVT::Other), Ops);
4107 ReplaceUses(
N, Dec);
4108 CurDAG->RemoveDeadNode(
N);
4124 unsigned Opc = Subtarget->isThumb() ?
4125 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4143 if (
ID == Intrinsic::loop_decrement_reg) {
4145 SDValue Size = CurDAG->getTargetConstant(
Int.getConstantOperandVal(3),
4150 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4151 CurDAG->getVTList(MVT::i32, MVT::Other),
4153 ReplaceUses(
Int.getNode(), LoopDec);
4157 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
4159 ReplaceUses(
N, LoopEnd);
4160 CurDAG->RemoveDeadNode(
N);
4161 CurDAG->RemoveDeadNode(InGlue.
getNode());
4162 CurDAG->RemoveDeadNode(
Int.getNode());
4167 bool SwitchEQNEToPLMI;
4168 SelectCMPZ(InGlue.
getNode(), SwitchEQNEToPLMI);
4169 InGlue =
N->getOperand(4);
4171 if (SwitchEQNEToPLMI) {
4184 SDValue Tmp2 = CurDAG->getTargetConstant(
CC, dl, MVT::i32);
4185 SDValue Ops[] = { N1, Tmp2, N3, Chain, InGlue };
4186 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
4189 if (
N->getNumValues() == 2) {
4191 ReplaceUses(
SDValue(
N, 1), InGlue);
4195 CurDAG->RemoveDeadNode(
N);
4205 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1).getNode());
4206 if (
C &&
C->getSExtValue() < 0 && Subtarget->isThumb()) {
4207 int64_t Addend = -
C->getSExtValue();
4214 if (Addend < 1<<8) {
4216 SDValue Ops[] = {
X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4217 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
4218 CurDAG->getRegister(0, MVT::i32) };
4219 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
4221 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4222 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
X,
4223 CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4224 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
4225 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
4230 CurDAG->MorphNodeTo(
N,
ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
4241 bool SwitchEQNEToPLMI;
4242 SelectCMPZ(InGlue.
getNode(), SwitchEQNEToPLMI);
4244 if (SwitchEQNEToPLMI) {
4257 SDValue NewARMcc = CurDAG->getConstant((
unsigned)
CC, dl, MVT::i32);
4258 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), NewARMcc,
4259 N->getOperand(3),
N->getOperand(4)};
4268 EVT VT =
N->getValueType(0);
4270 unsigned Opc64[] = {ARM::VZIPd8, ARM::VZIPd16, ARM::VTRNd32};
4271 unsigned Opc128[] = {ARM::VZIPq8, ARM::VZIPq16, ARM::VZIPq32};
4274 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4275 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4276 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4280 EVT VT =
N->getValueType(0);
4282 unsigned Opc64[] = {ARM::VUZPd8, ARM::VUZPd16, ARM::VTRNd32};
4283 unsigned Opc128[] = {ARM::VUZPq8, ARM::VUZPq16, ARM::VUZPq32};
4286 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4287 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4288 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4292 EVT VT =
N->getValueType(0);
4293 unsigned Opc64[] = {ARM::VTRNd8, ARM::VTRNd16, ARM::VTRNd32};
4294 unsigned Opc128[] = {ARM::VTRNq8, ARM::VTRNq16, ARM::VTRNq32};
4297 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4298 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4299 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4303 EVT VecVT =
N->getValueType(0);
4306 if (EltVT == MVT::f64) {
4307 assert(NumElts == 2 &&
"unexpected type for BUILD_VECTOR");
4309 N, createDRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4312 assert(EltVT == MVT::f32 &&
"unexpected type for BUILD_VECTOR");
4315 N, createSRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4318 assert(NumElts == 4 &&
"unexpected type for BUILD_VECTOR");
4320 createQuadSRegsNode(VecVT,
N->getOperand(0),
N->getOperand(1),
4321 N->getOperand(2),
N->getOperand(3)));
4326 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4328 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4330 SelectVLDDup(
N,
false,
false, 1, DOpcodes, QOpcodes);
4335 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4337 SelectVLDDup(
N,
false,
false, 2, Opcodes);
4342 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4343 ARM::VLD3DUPd16Pseudo,
4344 ARM::VLD3DUPd32Pseudo };
4345 SelectVLDDup(
N,
false,
false, 3, Opcodes);
4350 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4351 ARM::VLD4DUPd16Pseudo,
4352 ARM::VLD4DUPd32Pseudo };
4353 SelectVLDDup(
N,
false,
false, 4, Opcodes);
4358 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4359 ARM::VLD1DUPd16wb_fixed,
4360 ARM::VLD1DUPd32wb_fixed };
4361 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4362 ARM::VLD1DUPq16wb_fixed,
4363 ARM::VLD1DUPq32wb_fixed };
4364 SelectVLDDup(
N,
false,
true, 1, DOpcodes, QOpcodes);
4369 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4370 ARM::VLD2DUPd16wb_fixed,
4371 ARM::VLD2DUPd32wb_fixed,
4372 ARM::VLD1q64wb_fixed };
4373 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4374 ARM::VLD2DUPq16EvenPseudo,
4375 ARM::VLD2DUPq32EvenPseudo };
4376 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4377 ARM::VLD2DUPq16OddPseudoWB_fixed,
4378 ARM::VLD2DUPq32OddPseudoWB_fixed };
4379 SelectVLDDup(
N,
false,
true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4384 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4385 ARM::VLD3DUPd16Pseudo_UPD,
4386 ARM::VLD3DUPd32Pseudo_UPD,
4387 ARM::VLD1d64TPseudoWB_fixed };
4388 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4389 ARM::VLD3DUPq16EvenPseudo,
4390 ARM::VLD3DUPq32EvenPseudo };
4391 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4392 ARM::VLD3DUPq16OddPseudo_UPD,
4393 ARM::VLD3DUPq32OddPseudo_UPD };
4394 SelectVLDDup(
N,
false,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4399 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4400 ARM::VLD4DUPd16Pseudo_UPD,
4401 ARM::VLD4DUPd32Pseudo_UPD,
4402 ARM::VLD1d64QPseudoWB_fixed };
4403 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4404 ARM::VLD4DUPq16EvenPseudo,
4405 ARM::VLD4DUPq32EvenPseudo };
4406 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4407 ARM::VLD4DUPq16OddPseudo_UPD,
4408 ARM::VLD4DUPq32OddPseudo_UPD };
4409 SelectVLDDup(
N,
false,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4414 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4415 ARM::VLD1d16wb_fixed,
4416 ARM::VLD1d32wb_fixed,
4417 ARM::VLD1d64wb_fixed };
4418 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4419 ARM::VLD1q16wb_fixed,
4420 ARM::VLD1q32wb_fixed,
4421 ARM::VLD1q64wb_fixed };
4422 SelectVLD(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4427 if (Subtarget->hasNEON()) {
4428 static const uint16_t DOpcodes[] = {
4429 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4430 ARM::VLD1q64wb_fixed};
4431 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4432 ARM::VLD2q16PseudoWB_fixed,
4433 ARM::VLD2q32PseudoWB_fixed};
4434 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4436 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4437 ARM::MVE_VLD21_8_wb};
4438 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4439 ARM::MVE_VLD21_16_wb};
4440 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4441 ARM::MVE_VLD21_32_wb};
4442 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4443 SelectMVE_VLD(
N, 2, Opcodes,
true);
4449 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4450 ARM::VLD3d16Pseudo_UPD,
4451 ARM::VLD3d32Pseudo_UPD,
4452 ARM::VLD1d64TPseudoWB_fixed};
4453 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4454 ARM::VLD3q16Pseudo_UPD,
4455 ARM::VLD3q32Pseudo_UPD };
4456 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4457 ARM::VLD3q16oddPseudo_UPD,
4458 ARM::VLD3q32oddPseudo_UPD };
4459 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4464 if (Subtarget->hasNEON()) {
4465 static const uint16_t DOpcodes[] = {
4466 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4467 ARM::VLD1d64QPseudoWB_fixed};
4468 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4469 ARM::VLD4q16Pseudo_UPD,
4470 ARM::VLD4q32Pseudo_UPD};
4471 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4472 ARM::VLD4q16oddPseudo_UPD,
4473 ARM::VLD4q32oddPseudo_UPD};
4474 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4476 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4478 ARM::MVE_VLD43_8_wb};
4479 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4481 ARM::MVE_VLD43_16_wb};
4482 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4484 ARM::MVE_VLD43_32_wb};
4485 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4486 SelectMVE_VLD(
N, 4, Opcodes,
true);
4492 if (Subtarget->hasNEON()) {
4493 static const uint16_t DOpcodes[] = {
4494 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4495 ARM::VLD1q64wb_fixed};
4496 static const uint16_t QOpcodes[] = {
4497 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4498 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4499 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4506 if (Subtarget->hasNEON()) {
4507 static const uint16_t DOpcodes[] = {
4508 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4509 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4510 static const uint16_t QOpcodes0[] = {
4511 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4512 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4513 static const uint16_t QOpcodes1[] = {
4514 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4515 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4516 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4523 if (Subtarget->hasNEON()) {
4524 static const uint16_t DOpcodes[] = {
4525 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4526 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4527 static const uint16_t QOpcodes0[] = {
4528 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4529 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4530 static const uint16_t QOpcodes1[] = {
4531 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4532 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4533 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4540 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4541 ARM::VLD2LNd16Pseudo_UPD,
4542 ARM::VLD2LNd32Pseudo_UPD };
4543 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4544 ARM::VLD2LNq32Pseudo_UPD };
4545 SelectVLDSTLane(
N,
true,
true, 2, DOpcodes, QOpcodes);
4550 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4551 ARM::VLD3LNd16Pseudo_UPD,
4552 ARM::VLD3LNd32Pseudo_UPD };
4553 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4554 ARM::VLD3LNq32Pseudo_UPD };
4555 SelectVLDSTLane(
N,
true,
true, 3, DOpcodes, QOpcodes);
4560 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4561 ARM::VLD4LNd16Pseudo_UPD,
4562 ARM::VLD4LNd32Pseudo_UPD };
4563 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4564 ARM::VLD4LNq32Pseudo_UPD };
4565 SelectVLDSTLane(
N,
true,
true, 4, DOpcodes, QOpcodes);
4570 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4571 ARM::VST1d16wb_fixed,
4572 ARM::VST1d32wb_fixed,
4573 ARM::VST1d64wb_fixed };
4574 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4575 ARM::VST1q16wb_fixed,
4576 ARM::VST1q32wb_fixed,
4577 ARM::VST1q64wb_fixed };
4578 SelectVST(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4583 if (Subtarget->hasNEON()) {
4584 static const uint16_t DOpcodes[] = {
4585 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4586 ARM::VST1q64wb_fixed};
4587 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4588 ARM::VST2q16PseudoWB_fixed,
4589 ARM::VST2q32PseudoWB_fixed};
4590 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4597 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4598 ARM::VST3d16Pseudo_UPD,
4599 ARM::VST3d32Pseudo_UPD,
4600 ARM::VST1d64TPseudoWB_fixed};
4601 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4602 ARM::VST3q16Pseudo_UPD,
4603 ARM::VST3q32Pseudo_UPD };
4604 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4605 ARM::VST3q16oddPseudo_UPD,
4606 ARM::VST3q32oddPseudo_UPD };
4607 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4612 if (Subtarget->hasNEON()) {
4613 static const uint16_t DOpcodes[] = {
4614 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4615 ARM::VST1d64QPseudoWB_fixed};
4616 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4617 ARM::VST4q16Pseudo_UPD,
4618 ARM::VST4q32Pseudo_UPD};
4619 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4620 ARM::VST4q16oddPseudo_UPD,
4621 ARM::VST4q32oddPseudo_UPD};
4622 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4629 if (Subtarget->hasNEON()) {
4630 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4631 ARM::VST1q16wb_fixed,
4632 ARM::VST1q32wb_fixed,
4633 ARM::VST1q64wb_fixed};
4634 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4635 ARM::VST1d16QPseudoWB_fixed,
4636 ARM::VST1d32QPseudoWB_fixed,
4637 ARM::VST1d64QPseudoWB_fixed };
4638 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4645 if (Subtarget->hasNEON()) {
4646 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4647 ARM::VST1d16TPseudoWB_fixed,
4648 ARM::VST1d32TPseudoWB_fixed,
4649 ARM::VST1d64TPseudoWB_fixed };
4650 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4651 ARM::VST1q16LowTPseudo_UPD,
4652 ARM::VST1q32LowTPseudo_UPD,
4653 ARM::VST1q64LowTPseudo_UPD };
4654 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4655 ARM::VST1q16HighTPseudo_UPD,
4656 ARM::VST1q32HighTPseudo_UPD,
4657 ARM::VST1q64HighTPseudo_UPD };
4658 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4665 if (Subtarget->hasNEON()) {
4666 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4667 ARM::VST1d16QPseudoWB_fixed,
4668 ARM::VST1d32QPseudoWB_fixed,
4669 ARM::VST1d64QPseudoWB_fixed };
4670 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4671 ARM::VST1q16LowQPseudo_UPD,
4672 ARM::VST1q32LowQPseudo_UPD,
4673 ARM::VST1q64LowQPseudo_UPD };
4674 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4675 ARM::VST1q16HighQPseudo_UPD,
4676 ARM::VST1q32HighQPseudo_UPD,
4677 ARM::VST1q64HighQPseudo_UPD };
4678 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4684 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4685 ARM::VST2LNd16Pseudo_UPD,
4686 ARM::VST2LNd32Pseudo_UPD };
4687 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4688 ARM::VST2LNq32Pseudo_UPD };
4689 SelectVLDSTLane(
N,
false,
true, 2, DOpcodes, QOpcodes);
4694 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4695 ARM::VST3LNd16Pseudo_UPD,
4696 ARM::VST3LNd32Pseudo_UPD };
4697 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4698 ARM::VST3LNq32Pseudo_UPD };
4699 SelectVLDSTLane(
N,
false,
true, 3, DOpcodes, QOpcodes);
4704 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4705 ARM::VST4LNd16Pseudo_UPD,
4706 ARM::VST4LNd32Pseudo_UPD };
4707 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4708 ARM::VST4LNq32Pseudo_UPD };
4709 SelectVLDSTLane(
N,
false,
true, 4, DOpcodes, QOpcodes);
4715 unsigned IntNo =
N->getConstantOperandVal(1);
4720 case Intrinsic::arm_mrrc:
4721 case Intrinsic::arm_mrrc2: {
4726 if (Subtarget->isThumb())
4727 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4729 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4732 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(2), dl));
4733 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(3), dl));
4734 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(4), dl));
4739 if (Opc != ARM::MRRC2) {
4741 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4747 const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
4749 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4752 case Intrinsic::arm_ldaexd:
4753 case Intrinsic::arm_ldrexd: {
4756 SDValue MemAddr =
N->getOperand(2);
4757 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4759 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4760 unsigned NewOpc =
isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4761 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4764 std::vector<EVT> ResTys;
4766 ResTys.push_back(MVT::i32);
4767 ResTys.push_back(MVT::i32);
4769 ResTys.push_back(MVT::Untyped);
4770 ResTys.push_back(MVT::Other);
4774 CurDAG->getRegister(0, MVT::i32), Chain};
4775 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4778 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});