31#include "llvm/IR/IntrinsicsARM.h"
40#define DEBUG_TYPE "arm-isel"
41#define PASS_NAME "ARM Instruction Selection"
45 cl::desc(
"Disable isel of shifter-op"),
60 ARMDAGToDAGISel() =
delete;
76 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &dl) {
77 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
86 bool hasNoVMLxHazardUse(
SDNode *
N)
const;
87 bool isShifterOpProfitable(
const SDValue &Shift,
91 bool CheckProfitability =
true);
93 SDValue &
B,
bool CheckProfitability =
true);
97 return SelectRegShifterOperand(
N,
A,
B,
C,
false);
101 return SelectImmShifterOperand(
N,
A,
B,
false);
106 return SelectImmShifterOperand(
N,
A,
B,
false);
145 template <
unsigned Shift>
150 template <
unsigned Shift>
156 template <
unsigned Shift>
160 template <
unsigned Shift>
166 template<
int Min,
int Max>
169 inline bool is_so_imm(
unsigned Imm)
const {
173 inline bool is_so_imm_not(
unsigned Imm)
const {
177 inline bool is_t2_so_imm(
unsigned Imm)
const {
181 inline bool is_t2_so_imm_not(
unsigned Imm)
const {
186#include "ARMGenDAGISel.inc"
192 bool tryARMIndexedLoad(
SDNode *
N);
193 bool tryT1IndexedLoad(
SDNode *
N);
194 bool tryT2IndexedLoad(
SDNode *
N);
195 bool tryMVEIndexedLoad(
SDNode *
N);
206 void SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
214 void SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
221 void SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
222 unsigned NumVecs,
const uint16_t *DOpcodes,
226 template <
typename SDValueVector>
227 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
229 template <
typename SDValueVector>
230 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
233 template <
typename SDValueVector>
234 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc);
235 template <
typename SDValueVector>
236 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
EVT InactiveTy);
243 bool HasSaturationOperand);
263 size_t Stride,
size_t TySize);
279 void SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
280 const uint16_t *
const *Opcodes,
bool HasWriteback);
299 void SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
bool isUpdating,
300 unsigned NumVecs,
const uint16_t *DOpcodes,
301 const uint16_t *QOpcodes0 =
nullptr,
302 const uint16_t *QOpcodes1 =
nullptr);
307 bool tryInsertVectorElt(
SDNode *
N);
312 bool tryReadRegister(
SDNode *
N);
313 bool tryWriteRegister(
SDNode *
N);
317 void SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI);
319 void SelectCMP_SWAP(
SDNode *
N);
325 std::vector<SDValue> &OutOps)
override;
347 bool canExtractShiftFromMul(
const SDValue &
N,
unsigned MaxShift,
348 unsigned &PowerOfTwo,
SDValue &NewMulConst)
const;
360 ID,
std::make_unique<ARMDAGToDAGISel>(tm, OptLevel)) {}
364char ARMDAGToDAGISelLegacy::ID = 0;
371 if (
N->getOpcode() ==
ISD::Constant &&
N->getValueType(0) == MVT::i32) {
372 Imm =
N->getAsZExtVal();
388 return N->getOpcode() == Opc &&
397 int RangeMin,
int RangeMax,
398 int &ScaledConstant) {
399 assert(Scale > 0 &&
"Invalid scale!");
406 ScaledConstant = (int)
C->getZExtValue();
407 if ((ScaledConstant % Scale) != 0)
410 ScaledConstant /= Scale;
411 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
414void ARMDAGToDAGISel::PreprocessISelDAG() {
415 if (!Subtarget->hasV6T2Ops())
418 bool isThumb2 = Subtarget->isThumb();
435 unsigned And_imm = 0;
445 if (TZ != 1 && TZ != 2)
457 if (And_imm & (And_imm + 1))
462 unsigned Srl_imm = 0;
473 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
476 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
477 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
484 CurDAG->getConstant(Srl_imm + TZ,
SDLoc(Srl),
488 CurDAG->getConstant(And_imm,
SDLoc(Srl), MVT::i32));
490 N1, CurDAG->getConstant(TZ,
SDLoc(Srl), MVT::i32));
491 CurDAG->UpdateNodeOperands(&
N, N0, N1);
498bool ARMDAGToDAGISel::hasNoVMLxHazardUse(
SDNode *
N)
const {
499 if (OptLevel == CodeGenOptLevel::None)
502 if (!Subtarget->hasVMLxHazards())
511 if (
User->isMachineOpcode()) {
513 CurDAG->getSubtarget().getInstrInfo());
519 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
534 return TII->isFpMLxInstruction(Opcode);
540bool ARMDAGToDAGISel::isShifterOpProfitable(
const SDValue &Shift,
549 (ShAmt == 2 || (Subtarget->
isSwift() && ShAmt == 1));
552bool ARMDAGToDAGISel::canExtractShiftFromMul(
const SDValue &
N,
554 unsigned &PowerOfTwo,
561 if (!
N.hasOneUse())
return false;
564 if (!MulConst)
return false;
567 if (!MulConst->
hasOneUse())
return false;
569 if (MulConstVal == 0)
return false;
572 PowerOfTwo = MaxShift;
573 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
575 if (PowerOfTwo == 0)
return false;
579 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
580 NewMulConst = CurDAG->getConstant(NewMulConstVal,
SDLoc(
N), MVT::i32);
583 return NewCost < OldCost;
587 CurDAG->RepositionNode(
N.getNode()->getIterator(),
M.getNode());
591bool ARMDAGToDAGISel::SelectImmShifterOperand(
SDValue N,
594 bool CheckProfitability) {
601 unsigned PowerOfTwo = 0;
603 if (canExtractShiftFromMul(
N, 31, PowerOfTwo, NewMulConst)) {
606 replaceDAGValue(
N.getOperand(1), NewMulConst);
608 Opc = CurDAG->getTargetConstant(
620 BaseReg =
N.getOperand(0);
621 unsigned ShImmVal = 0;
623 if (!RHS)
return false;
624 ShImmVal =
RHS->getZExtValue() & 31;
630bool ARMDAGToDAGISel::SelectRegShifterOperand(
SDValue N,
634 bool CheckProfitability) {
644 BaseReg =
N.getOperand(0);
645 unsigned ShImmVal = 0;
647 if (RHS)
return false;
649 ShReg =
N.getOperand(1);
650 if (CheckProfitability && !isShifterOpProfitable(
N, ShOpcVal, ShImmVal))
662 return CurDAG->haveNoCommonBitsSet(
N, Parent->
getOperand(1));
666bool ARMDAGToDAGISel::SelectAddrModeImm12(
SDValue N,
673 !CurDAG->isBaseWithConstantOffset(
N)) {
676 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
677 Base = CurDAG->getTargetFrameIndex(
678 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
679 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
687 Base =
N.getOperand(0);
690 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
695 int RHSC = (int)
RHS->getSExtValue();
699 if (RHSC > -0x1000 && RHSC < 0x1000) {
700 Base =
N.getOperand(0);
702 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
703 Base = CurDAG->getTargetFrameIndex(
704 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
706 OffImm = CurDAG->getSignedTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
713 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
725 int RHSC = (int)
RHS->getZExtValue();
734 unsigned ShAmt =
Log2_32(RHSC);
747 !CurDAG->isBaseWithConstantOffset(
N))
754 -0x1000+1, 0x1000, RHSC))
764 Base =
N.getOperand(0);
771 dyn_cast<ConstantSDNode>(
N.getOperand(1).getOperand(1))) {
772 ShAmt = Sh->getZExtValue();
773 if (isShifterOpProfitable(
Offset, ShOpcVal, ShAmt))
774 Offset =
N.getOperand(1).getOperand(0);
787 N.getOperand(0).hasOneUse())) {
793 dyn_cast<ConstantSDNode>(
N.getOperand(0).getOperand(1))) {
794 ShAmt = Sh->getZExtValue();
795 if (isShifterOpProfitable(
N.getOperand(0), ShOpcVal, ShAmt)) {
796 Offset =
N.getOperand(0).getOperand(0);
797 Base =
N.getOperand(1);
811 unsigned PowerOfTwo = 0;
813 if (canExtractShiftFromMul(
Offset, 31, PowerOfTwo, NewMulConst)) {
815 replaceDAGValue(
Offset.getOperand(1), NewMulConst);
816 Offset = Handle.getValue();
829 unsigned Opcode =
Op->getOpcode();
831 ? cast<LoadSDNode>(
Op)->getAddressingMode()
832 : cast<StoreSDNode>(
Op)->getAddressingMode();
846 ShAmt = Sh->getZExtValue();
847 if (isShifterOpProfitable(
N, ShOpcVal, ShAmt))
865 unsigned Opcode =
Op->getOpcode();
867 ? cast<LoadSDNode>(
Op)->getAddressingMode()
868 : cast<StoreSDNode>(
Op)->getAddressingMode();
874 Offset = CurDAG->getRegister(0, MVT::i32);
875 Opc = CurDAG->getSignedTargetConstant(Val,
SDLoc(
Op), MVT::i32);
885 unsigned Opcode =
Op->getOpcode();
887 ? cast<LoadSDNode>(
Op)->getAddressingMode()
888 : cast<StoreSDNode>(
Op)->getAddressingMode();
893 Offset = CurDAG->getRegister(0, MVT::i32);
908bool ARMDAGToDAGISel::SelectAddrMode3(
SDValue N,
913 Base =
N.getOperand(0);
920 if (!CurDAG->isBaseWithConstantOffset(
N)) {
923 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
924 Base = CurDAG->getTargetFrameIndex(
925 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
927 Offset = CurDAG->getRegister(0, MVT::i32);
936 -256 + 1, 256, RHSC)) {
937 Base =
N.getOperand(0);
939 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
940 Base = CurDAG->getTargetFrameIndex(
941 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
943 Offset = CurDAG->getRegister(0, MVT::i32);
955 Base =
N.getOperand(0);
964 unsigned Opcode =
Op->getOpcode();
966 ? cast<LoadSDNode>(
Op)->getAddressingMode()
967 : cast<StoreSDNode>(
Op)->getAddressingMode();
972 Offset = CurDAG->getRegister(0, MVT::i32);
986 if (!CurDAG->isBaseWithConstantOffset(
N)) {
989 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
990 Base = CurDAG->getTargetFrameIndex(
991 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
996 Base =
N.getOperand(0);
1005 const int Scale =
FP16 ? 2 : 4;
1008 Base =
N.getOperand(0);
1010 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1011 Base = CurDAG->getTargetFrameIndex(
1012 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1043bool ARMDAGToDAGISel::SelectAddrMode5(
SDValue N,
1048bool ARMDAGToDAGISel::SelectAddrMode5FP16(
SDValue N,
1057 unsigned Alignment = 0;
1059 MemSDNode *MemN = cast<MemSDNode>(Parent);
1061 if (isa<LSBaseSDNode>(MemN) ||
1069 if (MMOAlign.
value() >= MemSize && MemSize > 1)
1070 Alignment = MemSize;
1078 Align = CurDAG->getTargetConstant(Alignment,
SDLoc(
N), MVT::i32);
1091 Offset = CurDAG->getRegister(0, MVT::i32);
1096bool ARMDAGToDAGISel::SelectAddrModePC(
SDValue N,
1121 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
1122 return C->getSExtValue() < 0 &&
C->getSExtValue() >= -255;
1129 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N)) {
1137 Base =
N.getOperand(0);
1146 return SelectThumbAddrModeRRSext(
N,
Base,
Offset);
1150ARMDAGToDAGISel::SelectThumbAddrModeImm5S(
SDValue N,
unsigned Scale,
1154 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1158 if (!CurDAG->isBaseWithConstantOffset(
N)) {
1166 Base =
N.getOperand(0);
1171 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1178 Base =
N.getOperand(0);
1179 OffImm = CurDAG->getSignedTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1190 return SelectThumbAddrModeImm5S(
N, 4,
Base, OffImm);
1196 return SelectThumbAddrModeImm5S(
N, 2,
Base, OffImm);
1202 return SelectThumbAddrModeImm5S(
N, 1,
Base, OffImm);
1205bool ARMDAGToDAGISel::SelectThumbAddrModeSP(
SDValue N,
1208 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1214 Base = CurDAG->getTargetFrameIndex(
1215 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1216 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1220 if (!CurDAG->isBaseWithConstantOffset(
N))
1227 Base =
N.getOperand(0);
1228 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1239 Base = CurDAG->getTargetFrameIndex(
1240 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1241 OffImm = CurDAG->getSignedTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1251template <
unsigned Shift>
1254 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1258 Base =
N.getOperand(0);
1261 OffImm = CurDAG->getSignedTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
1269 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1279bool ARMDAGToDAGISel::SelectT2AddrModeImm12(
SDValue N,
1285 !CurDAG->isBaseWithConstantOffset(
N)) {
1288 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1289 Base = CurDAG->getTargetFrameIndex(
1290 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1291 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1299 Base =
N.getOperand(0);
1304 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1308 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1309 if (SelectT2AddrModeImm8(
N,
Base, OffImm))
1313 int RHSC = (int)
RHS->getZExtValue();
1317 if (RHSC >= 0 && RHSC < 0x1000) {
1318 Base =
N.getOperand(0);
1320 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1321 Base = CurDAG->getTargetFrameIndex(
1322 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1324 OffImm = CurDAG->getSignedTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1331 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1335template <
unsigned Shift>
1338 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1341 Base =
N.getOperand(0);
1343 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1344 Base = CurDAG->getTargetFrameIndex(
1345 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1350 OffImm = CurDAG->getSignedTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
1358 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1362bool ARMDAGToDAGISel::SelectT2AddrModeImm8(
SDValue N,
1366 !CurDAG->isBaseWithConstantOffset(
N))
1369 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1370 int RHSC = (int)
RHS->getSExtValue();
1374 if ((RHSC >= -255) && (RHSC < 0)) {
1375 Base =
N.getOperand(0);
1377 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1378 Base = CurDAG->getTargetFrameIndex(
1379 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1381 OffImm = CurDAG->getSignedTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1391 unsigned Opcode =
Op->getOpcode();
1393 ? cast<LoadSDNode>(
Op)->getAddressingMode()
1394 : cast<StoreSDNode>(
Op)->getAddressingMode();
1398 ? CurDAG->getSignedTargetConstant(RHSC,
SDLoc(
N), MVT::i32)
1399 : CurDAG->getSignedTargetConstant(-RHSC,
SDLoc(
N), MVT::i32);
1406template <
unsigned Shift>
1409 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1413 Base =
N.getOperand(0);
1415 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1416 Base = CurDAG->getTargetFrameIndex(
1417 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1422 OffImm = CurDAG->getSignedTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
1430 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1434template <
unsigned Shift>
1437 return SelectT2AddrModeImm7Offset(
Op,
N, OffImm, Shift);
1443 unsigned Opcode =
Op->getOpcode();
1447 AM = cast<LoadSDNode>(
Op)->getAddressingMode();
1450 AM = cast<StoreSDNode>(
Op)->getAddressingMode();
1453 AM = cast<MaskedLoadSDNode>(
Op)->getAddressingMode();
1456 AM = cast<MaskedStoreSDNode>(
Op)->getAddressingMode();
1466 ? CurDAG->getSignedTargetConstant(RHSC * (1 << Shift),
1468 : CurDAG->getSignedTargetConstant(-RHSC * (1 << Shift),
1475template <
int Min,
int Max>
1476bool ARMDAGToDAGISel::SelectImmediateInRange(
SDValue N,
SDValue &OffImm) {
1479 OffImm = CurDAG->getSignedTargetConstant(Val,
SDLoc(
N), MVT::i32);
1485bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(
SDValue N,
1489 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N))
1493 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1494 int RHSC = (int)
RHS->getZExtValue();
1495 if (RHSC >= 0 && RHSC < 0x1000)
1497 else if (RHSC < 0 && RHSC >= -255)
1503 Base =
N.getOperand(0);
1504 OffReg =
N.getOperand(1);
1518 ShAmt = Sh->getZExtValue();
1519 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1530 unsigned PowerOfTwo = 0;
1532 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1534 replaceDAGValue(OffReg.
getOperand(1), NewMulConst);
1540 ShImm = CurDAG->getTargetConstant(ShAmt,
SDLoc(
N), MVT::i32);
1550 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1552 if (
N.getOpcode() !=
ISD::ADD || !CurDAG->isBaseWithConstantOffset(
N))
1560 if (RHSC > 1020 || RHSC % 4 != 0)
1563 Base =
N.getOperand(0);
1565 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1566 Base = CurDAG->getTargetFrameIndex(
1567 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1570 OffImm = CurDAG->getTargetConstant(RHSC/4,
SDLoc(
N), MVT::i32);
1581void ARMDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
1583 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {
MemOp});
1586bool ARMDAGToDAGISel::tryARMIndexedLoad(
SDNode *
N) {
1592 EVT LoadedVT =
LD->getMemoryVT();
1595 unsigned Opcode = 0;
1597 if (LoadedVT == MVT::i32 && isPre &&
1598 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1599 Opcode = ARM::LDR_PRE_IMM;
1601 }
else if (LoadedVT == MVT::i32 && !isPre &&
1602 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1603 Opcode = ARM::LDR_POST_IMM;
1605 }
else if (LoadedVT == MVT::i32 &&
1606 SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1607 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1610 }
else if (LoadedVT == MVT::i16 &&
1611 SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1614 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1615 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1616 }
else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1618 if (SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1620 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1624 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1626 Opcode = ARM::LDRB_PRE_IMM;
1627 }
else if (!isPre &&
1628 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1630 Opcode = ARM::LDRB_POST_IMM;
1631 }
else if (SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1633 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1639 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1643 CurDAG->getRegister(0, MVT::i32), Chain };
1644 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1646 transferMemOperands(
N, New);
1647 ReplaceNode(
N, New);
1653 CurDAG->getRegister(0, MVT::i32), Chain };
1654 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1656 transferMemOperands(
N, New);
1657 ReplaceNode(
N, New);
1665bool ARMDAGToDAGISel::tryT1IndexedLoad(
SDNode *
N) {
1667 EVT LoadedVT =
LD->getMemoryVT();
1673 auto *COffs = dyn_cast<ConstantSDNode>(
LD->getOffset());
1674 if (!COffs || COffs->getZExtValue() != 4)
1684 CurDAG->getRegister(0, MVT::i32), Chain };
1685 SDNode *
New = CurDAG->getMachineNode(ARM::tLDR_postidx,
SDLoc(
N), MVT::i32,
1686 MVT::i32, MVT::Other, Ops);
1687 transferMemOperands(
N, New);
1688 ReplaceNode(
N, New);
1692bool ARMDAGToDAGISel::tryT2IndexedLoad(
SDNode *
N) {
1698 EVT LoadedVT =
LD->getMemoryVT();
1702 unsigned Opcode = 0;
1704 if (SelectT2AddrModeImm8Offset(
N,
LD->getOffset(),
Offset)) {
1707 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1711 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1713 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1718 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1720 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1732 CurDAG->getRegister(0, MVT::i32), Chain };
1733 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1735 transferMemOperands(
N, New);
1736 ReplaceNode(
N, New);
1743bool ARMDAGToDAGISel::tryMVEIndexedLoad(
SDNode *
N) {
1745 unsigned Opcode = 0;
1746 bool isSExtLd, isPre;
1756 LoadedVT =
LD->getMemoryVT();
1760 Chain =
LD->getChain();
1763 Alignment =
LD->getAlign();
1767 PredReg = CurDAG->getRegister(0, MVT::i32);
1772 LoadedVT =
LD->getMemoryVT();
1776 Chain =
LD->getChain();
1779 Alignment =
LD->getAlign();
1783 PredReg =
LD->getMask();
1790 bool CanChangeType = Subtarget->
isLittle() && !isa<MaskedLoadSDNode>(
N);
1793 if (Alignment >=
Align(2) && LoadedVT == MVT::v4i16 &&
1794 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1)) {
1796 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1798 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1799 }
else if (LoadedVT == MVT::v8i8 &&
1800 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1802 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1804 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1805 }
else if (LoadedVT == MVT::v4i8 &&
1806 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1808 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1810 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1811 }
else if (Alignment >=
Align(4) &&
1812 (CanChangeType || LoadedVT == MVT::v4i32 ||
1813 LoadedVT == MVT::v4f32) &&
1814 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 2))
1815 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1816 else if (Alignment >=
Align(2) &&
1817 (CanChangeType || LoadedVT == MVT::v8i16 ||
1818 LoadedVT == MVT::v8f16) &&
1819 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1))
1820 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1821 else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
1822 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0))
1823 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1829 CurDAG->getTargetConstant(Pred,
SDLoc(
N), MVT::i32),
1831 CurDAG->getRegister(0, MVT::i32),
1834 N->getValueType(0), MVT::Other, Ops);
1835 transferMemOperands(
N, New);
1839 CurDAG->RemoveDeadNode(
N);
1847 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1848 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1849 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1850 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1851 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1858 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1859 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1860 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1861 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1862 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1868 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1870 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1871 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1872 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1873 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1879 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1881 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1882 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1883 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1884 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1892 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1893 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1894 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1895 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1896 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1897 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1898 V2, SubReg2, V3, SubReg3 };
1899 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1906 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1908 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1909 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1910 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1911 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1912 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1913 V2, SubReg2, V3, SubReg3 };
1914 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1921 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1923 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1924 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1925 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1926 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1927 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1928 V2, SubReg2, V3, SubReg3 };
1929 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1936 unsigned NumVecs,
bool is64BitVector) {
1937 unsigned NumRegs = NumVecs;
1938 if (!is64BitVector && NumVecs < 3)
1941 unsigned Alignment =
Align->getAsZExtVal();
1942 if (Alignment >= 32 && NumRegs == 4)
1944 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1946 else if (Alignment >= 8)
1951 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1957 default:
return false;
1958 case ARM::VLD1d8wb_fixed :
return true;
1959 case ARM::VLD1d16wb_fixed :
return true;
1960 case ARM::VLD1d64Qwb_fixed :
return true;
1961 case ARM::VLD1d32wb_fixed :
return true;
1962 case ARM::VLD1d64wb_fixed :
return true;
1963 case ARM::VLD1d8TPseudoWB_fixed :
return true;
1964 case ARM::VLD1d16TPseudoWB_fixed :
return true;
1965 case ARM::VLD1d32TPseudoWB_fixed :
return true;
1966 case ARM::VLD1d64TPseudoWB_fixed :
return true;
1967 case ARM::VLD1d8QPseudoWB_fixed :
return true;
1968 case ARM::VLD1d16QPseudoWB_fixed :
return true;
1969 case ARM::VLD1d32QPseudoWB_fixed :
return true;
1970 case ARM::VLD1d64QPseudoWB_fixed :
return true;
1971 case ARM::VLD1q8wb_fixed :
return true;
1972 case ARM::VLD1q16wb_fixed :
return true;
1973 case ARM::VLD1q32wb_fixed :
return true;
1974 case ARM::VLD1q64wb_fixed :
return true;
1975 case ARM::VLD1DUPd8wb_fixed :
return true;
1976 case ARM::VLD1DUPd16wb_fixed :
return true;
1977 case ARM::VLD1DUPd32wb_fixed :
return true;
1978 case ARM::VLD1DUPq8wb_fixed :
return true;
1979 case ARM::VLD1DUPq16wb_fixed :
return true;
1980 case ARM::VLD1DUPq32wb_fixed :
return true;
1981 case ARM::VLD2d8wb_fixed :
return true;
1982 case ARM::VLD2d16wb_fixed :
return true;
1983 case ARM::VLD2d32wb_fixed :
return true;
1984 case ARM::VLD2q8PseudoWB_fixed :
return true;
1985 case ARM::VLD2q16PseudoWB_fixed :
return true;
1986 case ARM::VLD2q32PseudoWB_fixed :
return true;
1987 case ARM::VLD2DUPd8wb_fixed :
return true;
1988 case ARM::VLD2DUPd16wb_fixed :
return true;
1989 case ARM::VLD2DUPd32wb_fixed :
return true;
1990 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return true;
1991 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return true;
1992 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return true;
1999 default:
return false;
2000 case ARM::VST1d8wb_fixed :
return true;
2001 case ARM::VST1d16wb_fixed :
return true;
2002 case ARM::VST1d32wb_fixed :
return true;
2003 case ARM::VST1d64wb_fixed :
return true;
2004 case ARM::VST1q8wb_fixed :
return true;
2005 case ARM::VST1q16wb_fixed :
return true;
2006 case ARM::VST1q32wb_fixed :
return true;
2007 case ARM::VST1q64wb_fixed :
return true;
2008 case ARM::VST1d8TPseudoWB_fixed :
return true;
2009 case ARM::VST1d16TPseudoWB_fixed :
return true;
2010 case ARM::VST1d32TPseudoWB_fixed :
return true;
2011 case ARM::VST1d64TPseudoWB_fixed :
return true;
2012 case ARM::VST1d8QPseudoWB_fixed :
return true;
2013 case ARM::VST1d16QPseudoWB_fixed :
return true;
2014 case ARM::VST1d32QPseudoWB_fixed :
return true;
2015 case ARM::VST1d64QPseudoWB_fixed :
return true;
2016 case ARM::VST2d8wb_fixed :
return true;
2017 case ARM::VST2d16wb_fixed :
return true;
2018 case ARM::VST2d32wb_fixed :
return true;
2019 case ARM::VST2q8PseudoWB_fixed :
return true;
2020 case ARM::VST2q16PseudoWB_fixed :
return true;
2021 case ARM::VST2q32PseudoWB_fixed :
return true;
2029 &&
"Incorrect fixed stride updating instruction.");
2032 case ARM::VLD1d8wb_fixed:
return ARM::VLD1d8wb_register;
2033 case ARM::VLD1d16wb_fixed:
return ARM::VLD1d16wb_register;
2034 case ARM::VLD1d32wb_fixed:
return ARM::VLD1d32wb_register;
2035 case ARM::VLD1d64wb_fixed:
return ARM::VLD1d64wb_register;
2036 case ARM::VLD1q8wb_fixed:
return ARM::VLD1q8wb_register;
2037 case ARM::VLD1q16wb_fixed:
return ARM::VLD1q16wb_register;
2038 case ARM::VLD1q32wb_fixed:
return ARM::VLD1q32wb_register;
2039 case ARM::VLD1q64wb_fixed:
return ARM::VLD1q64wb_register;
2040 case ARM::VLD1d64Twb_fixed:
return ARM::VLD1d64Twb_register;
2041 case ARM::VLD1d64Qwb_fixed:
return ARM::VLD1d64Qwb_register;
2042 case ARM::VLD1d8TPseudoWB_fixed:
return ARM::VLD1d8TPseudoWB_register;
2043 case ARM::VLD1d16TPseudoWB_fixed:
return ARM::VLD1d16TPseudoWB_register;
2044 case ARM::VLD1d32TPseudoWB_fixed:
return ARM::VLD1d32TPseudoWB_register;
2045 case ARM::VLD1d64TPseudoWB_fixed:
return ARM::VLD1d64TPseudoWB_register;
2046 case ARM::VLD1d8QPseudoWB_fixed:
return ARM::VLD1d8QPseudoWB_register;
2047 case ARM::VLD1d16QPseudoWB_fixed:
return ARM::VLD1d16QPseudoWB_register;
2048 case ARM::VLD1d32QPseudoWB_fixed:
return ARM::VLD1d32QPseudoWB_register;
2049 case ARM::VLD1d64QPseudoWB_fixed:
return ARM::VLD1d64QPseudoWB_register;
2050 case ARM::VLD1DUPd8wb_fixed :
return ARM::VLD1DUPd8wb_register;
2051 case ARM::VLD1DUPd16wb_fixed :
return ARM::VLD1DUPd16wb_register;
2052 case ARM::VLD1DUPd32wb_fixed :
return ARM::VLD1DUPd32wb_register;
2053 case ARM::VLD1DUPq8wb_fixed :
return ARM::VLD1DUPq8wb_register;
2054 case ARM::VLD1DUPq16wb_fixed :
return ARM::VLD1DUPq16wb_register;
2055 case ARM::VLD1DUPq32wb_fixed :
return ARM::VLD1DUPq32wb_register;
2056 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return ARM::VLD2DUPq8OddPseudoWB_register;
2057 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return ARM::VLD2DUPq16OddPseudoWB_register;
2058 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return ARM::VLD2DUPq32OddPseudoWB_register;
2060 case ARM::VST1d8wb_fixed:
return ARM::VST1d8wb_register;
2061 case ARM::VST1d16wb_fixed:
return ARM::VST1d16wb_register;
2062 case ARM::VST1d32wb_fixed:
return ARM::VST1d32wb_register;
2063 case ARM::VST1d64wb_fixed:
return ARM::VST1d64wb_register;
2064 case ARM::VST1q8wb_fixed:
return ARM::VST1q8wb_register;
2065 case ARM::VST1q16wb_fixed:
return ARM::VST1q16wb_register;
2066 case ARM::VST1q32wb_fixed:
return ARM::VST1q32wb_register;
2067 case ARM::VST1q64wb_fixed:
return ARM::VST1q64wb_register;
2068 case ARM::VST1d8TPseudoWB_fixed:
return ARM::VST1d8TPseudoWB_register;
2069 case ARM::VST1d16TPseudoWB_fixed:
return ARM::VST1d16TPseudoWB_register;
2070 case ARM::VST1d32TPseudoWB_fixed:
return ARM::VST1d32TPseudoWB_register;
2071 case ARM::VST1d64TPseudoWB_fixed:
return ARM::VST1d64TPseudoWB_register;
2072 case ARM::VST1d8QPseudoWB_fixed:
return ARM::VST1d8QPseudoWB_register;
2073 case ARM::VST1d16QPseudoWB_fixed:
return ARM::VST1d16QPseudoWB_register;
2074 case ARM::VST1d32QPseudoWB_fixed:
return ARM::VST1d32QPseudoWB_register;
2075 case ARM::VST1d64QPseudoWB_fixed:
return ARM::VST1d64QPseudoWB_register;
2077 case ARM::VLD2d8wb_fixed:
return ARM::VLD2d8wb_register;
2078 case ARM::VLD2d16wb_fixed:
return ARM::VLD2d16wb_register;
2079 case ARM::VLD2d32wb_fixed:
return ARM::VLD2d32wb_register;
2080 case ARM::VLD2q8PseudoWB_fixed:
return ARM::VLD2q8PseudoWB_register;
2081 case ARM::VLD2q16PseudoWB_fixed:
return ARM::VLD2q16PseudoWB_register;
2082 case ARM::VLD2q32PseudoWB_fixed:
return ARM::VLD2q32PseudoWB_register;
2084 case ARM::VST2d8wb_fixed:
return ARM::VST2d8wb_register;
2085 case ARM::VST2d16wb_fixed:
return ARM::VST2d16wb_register;
2086 case ARM::VST2d32wb_fixed:
return ARM::VST2d32wb_register;
2087 case ARM::VST2q8PseudoWB_fixed:
return ARM::VST2q8PseudoWB_register;
2088 case ARM::VST2q16PseudoWB_fixed:
return ARM::VST2q16PseudoWB_register;
2089 case ARM::VST2q32PseudoWB_fixed:
return ARM::VST2q32PseudoWB_register;
2091 case ARM::VLD2DUPd8wb_fixed:
return ARM::VLD2DUPd8wb_register;
2092 case ARM::VLD2DUPd16wb_fixed:
return ARM::VLD2DUPd16wb_register;
2093 case ARM::VLD2DUPd32wb_fixed:
return ARM::VLD2DUPd32wb_register;
2102 auto C = dyn_cast<ConstantSDNode>(Inc);
2106void ARMDAGToDAGISel::SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2110 assert(Subtarget->hasNEON());
2111 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLD NumVecs out-of-range");
2115 bool IsIntrinsic = !isUpdating;
2117 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2118 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2122 EVT VT =
N->getValueType(0);
2124 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2152 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2157 std::vector<EVT> ResTys;
2158 ResTys.push_back(ResTy);
2160 ResTys.push_back(MVT::i32);
2161 ResTys.push_back(MVT::Other);
2164 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2169 if (is64BitVector || NumVecs <= 2) {
2170 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2175 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2191 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2201 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2202 const SDValue OpsA[] = { MemAddr,
Align, Reg0, ImplDef, Pred, Reg0, Chain };
2204 ResTy, AddrTy, MVT::Other, OpsA);
2211 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2213 "only constant post-increment update allowed for VLD3/4");
2221 VLd = CurDAG->getMachineNode(QOpcodes1[
OpcodeIndex], dl, ResTys, Ops);
2226 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {
MemOp});
2229 ReplaceNode(
N, VLd);
2235 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2236 ARM::qsub_3 == ARM::qsub_0 + 3,
2237 "Unexpected subreg numbering");
2238 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2239 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2241 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2245 CurDAG->RemoveDeadNode(
N);
2248void ARMDAGToDAGISel::SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2252 assert(Subtarget->hasNEON());
2253 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VST NumVecs out-of-range");
2257 bool IsIntrinsic = !isUpdating;
2259 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2260 unsigned Vec0Idx = 3;
2261 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2267 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2269 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2293 std::vector<EVT> ResTys;
2295 ResTys.push_back(MVT::i32);
2296 ResTys.push_back(MVT::Other);
2299 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2303 if (is64BitVector || NumVecs <= 2) {
2306 SrcReg =
N->getOperand(Vec0Idx);
2307 }
else if (is64BitVector) {
2309 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2310 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2312 SrcReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2318 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2319 :
N->getOperand(Vec0Idx + 3);
2320 SrcReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2324 SDValue Q0 =
N->getOperand(Vec0Idx);
2325 SDValue Q1 =
N->getOperand(Vec0Idx + 1);
2326 SrcReg =
SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2329 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2334 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2352 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2355 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {
MemOp});
2357 ReplaceNode(
N, VSt);
2365 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2366 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2369 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2370 :
N->getOperand(Vec0Idx + 3);
2371 SDValue RegSeq =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2375 const SDValue OpsA[] = { MemAddr,
Align, Reg0, RegSeq, Pred, Reg0, Chain };
2379 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {
MemOp});
2386 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2388 "only constant post-increment update allowed for VST3/4");
2398 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {
MemOp});
2399 ReplaceNode(
N, VStB);
2402void ARMDAGToDAGISel::SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
2406 assert(Subtarget->hasNEON());
2407 assert(NumVecs >=2 && NumVecs <= 4 &&
"VLDSTLane NumVecs out-of-range");
2411 bool IsIntrinsic = !isUpdating;
2413 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2414 unsigned Vec0Idx = 3;
2415 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2421 unsigned Lane =
N->getConstantOperandVal(Vec0Idx + NumVecs);
2422 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2425 unsigned Alignment = 0;
2427 Alignment =
Align->getAsZExtVal();
2429 if (Alignment > NumBytes)
2430 Alignment = NumBytes;
2431 if (Alignment < 8 && Alignment < NumBytes)
2434 Alignment = (Alignment & -Alignment);
2438 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2458 std::vector<EVT> ResTys;
2460 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2464 MVT::i64, ResTyElts));
2467 ResTys.push_back(MVT::i32);
2468 ResTys.push_back(MVT::Other);
2471 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2477 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2480 Ops.
push_back(IsImmUpdate ? Reg0 : Inc);
2484 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2485 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2488 SuperReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2490 SuperReg =
SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2494 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2495 :
N->getOperand(Vec0Idx + 3);
2497 SuperReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2499 SuperReg =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2507 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2509 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2510 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {
MemOp});
2512 ReplaceNode(
N, VLdLn);
2518 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2519 ARM::qsub_3 == ARM::qsub_0 + 3,
2520 "Unexpected subreg numbering");
2521 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2522 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2524 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2528 CurDAG->RemoveDeadNode(
N);
2531template <
typename SDValueVector>
2532void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2534 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2535 Ops.push_back(PredicateMask);
2536 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2539template <
typename SDValueVector>
2540void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2543 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2544 Ops.push_back(PredicateMask);
2545 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2546 Ops.push_back(Inactive);
2549template <
typename SDValueVector>
2550void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc) {
2551 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2552 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2553 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2556template <
typename SDValueVector>
2557void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2559 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2560 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2561 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2563 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2572 switch (
N->getValueType(1).getVectorElementType().getSizeInBits()) {
2574 Opcode = Opcodes[0];
2577 Opcode = Opcodes[1];
2585 int32_t ImmValue =
N->getConstantOperandVal(3);
2586 Ops.
push_back(getI32Imm(ImmValue, Loc));
2589 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2591 AddEmptyMVEPredicateToOps(Ops, Loc);
2604 transferMemOperands(
N, New);
2605 CurDAG->RemoveDeadNode(
N);
2610 bool HasSaturationOperand) {
2620 int32_t ImmValue =
N->getConstantOperandVal(3);
2621 Ops.
push_back(getI32Imm(ImmValue, Loc));
2627 if (HasSaturationOperand) {
2628 int32_t SatOp =
N->getConstantOperandVal(4);
2629 int SatBit = (SatOp == 64 ? 0 : 1);
2636 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
2638 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2641void ARMDAGToDAGISel::SelectMVE_VADCSBC(
SDNode *
N,
uint16_t OpcodeWithCarry,
2652 Ops.
push_back(
N->getOperand(FirstInputOp + 1));
2653 SDValue CarryIn =
N->getOperand(FirstInputOp + 2);
2654 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2657 if (CarryInConstant &&
2658 (CarryInConstant->
getZExtValue() & CarryMask) == CarryExpected) {
2659 Opcode = OpcodeWithNoCarry;
2662 Opcode = OpcodeWithCarry;
2666 AddMVEPredicateToOps(Ops, Loc,
2667 N->getOperand(FirstInputOp + 3),
2668 N->getOperand(FirstInputOp - 1));
2670 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2672 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2683 int32_t ImmValue =
N->getConstantOperandVal(3);
2684 Ops.
push_back(getI32Imm(ImmValue, Loc));
2687 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2689 AddEmptyMVEPredicateToOps(Ops, Loc);
2691 CurDAG->SelectNodeTo(
N, ARM::MVE_VSHLC,
N->getVTList(),
ArrayRef(Ops));
2695 assert(isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant");
2705 size_t Stride,
size_t TySize) {
2706 assert(TySize < Stride &&
"Invalid TySize");
2712 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2714 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2717 auto OpIsZero = [
N](
size_t OpNo) {
2723 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2725 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2727 Opcodes += 4 * Stride;
2729 Opcodes += 2 * Stride;
2746 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(8));
2748 AddEmptyMVEPredicateToOps(Ops, Loc);
2750 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2756 EVT VecTy =
N->getOperand(6).getValueType();
2769 SelectBaseMVE_VMLLDAV(
N,
Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2776 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2778 "bad vector element size");
2779 SelectBaseMVE_VMLLDAV(
N,
Predicated, OpcodesS, OpcodesU, 1, 0);
2782void ARMDAGToDAGISel::SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
2784 bool HasWriteback) {
2785 EVT VT =
N->getValueType(0);
2791 OurOpcodes = Opcodes[0];
2794 OurOpcodes = Opcodes[1];
2797 OurOpcodes = Opcodes[2];
2805 unsigned PtrOperand = HasWriteback ? 1 : 2;
2808 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2811 for (
unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2812 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2814 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2821 ResultTys = {DataTy, MVT::i32, MVT::Other};
2822 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2824 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2828 for (i = 0; i < NumVecs; i++)
2830 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2835 CurDAG->RemoveDeadNode(
N);
2838void ARMDAGToDAGISel::SelectMVE_VxDUP(
SDNode *
N,
const uint16_t *Opcodes,
2840 EVT VT =
N->getValueType(0);
2846 Opcode = Opcodes[0];
2849 Opcode = Opcodes[1];
2852 Opcode = Opcodes[2];
2863 Inactive =
N->getOperand(OpIdx++);
2869 SDValue ImmOp =
N->getOperand(OpIdx++);
2871 Ops.
push_back(getI32Imm(ImmValue, Loc));
2874 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(OpIdx), Inactive);
2876 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2878 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2882 size_t NumExtraOps,
bool HasAccum) {
2883 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2890 SDValue ImmCorpoc =
N->getOperand(OpIdx++);
2892 Ops.
push_back(getI32Imm(ImmCoprocVal, Loc));
2897 SDValue AccLo =
N->getOperand(OpIdx++);
2898 SDValue AccHi =
N->getOperand(OpIdx++);
2905 for (
size_t I = 0;
I < NumExtraOps;
I++)
2916 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2922 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops);
2929 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2933 for (
size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2934 if (
SDValue(
N, ResIdx).use_empty())
2936 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2937 MVT::i32, ResultPair);
2941 CurDAG->RemoveDeadNode(
N);
2944void ARMDAGToDAGISel::SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
2945 bool isUpdating,
unsigned NumVecs,
2949 assert(Subtarget->hasNEON());
2950 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLDDup NumVecs out-of-range");
2954 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2955 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2959 EVT VT =
N->getValueType(0);
2962 unsigned Alignment = 0;
2964 Alignment =
Align->getAsZExtVal();
2966 if (Alignment > NumBytes)
2967 Alignment = NumBytes;
2968 if (Alignment < 8 && Alignment < NumBytes)
2971 Alignment = (Alignment & -Alignment);
2975 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2997 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
3002 std::vector<EVT> ResTys;
3003 ResTys.push_back(ResTy);
3005 ResTys.push_back(MVT::i32);
3006 ResTys.push_back(MVT::Other);
3009 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3014 unsigned Opc = is64BitVector ? DOpcodes[
OpcodeIndex]
3030 if (is64BitVector || NumVecs == 1) {
3034 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3035 const SDValue OpsA[] = {MemAddr,
Align, ImplDef, Pred, Reg0, Chain};
3046 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3050 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {
MemOp});
3057 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7,
"Unexpected subreg numbering");
3058 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3059 for (
unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3061 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3067 CurDAG->RemoveDeadNode(
N);
3070bool ARMDAGToDAGISel::tryInsertVectorElt(
SDNode *
N) {
3071 if (!Subtarget->hasMVEIntegerOps())
3085 (VT != MVT::v8f16 && VT != MVT::v8i16) || (Ins2.
getValueType() != VT))
3090 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3117 ExtractLane1 == ExtractLane2 + 1) {
3118 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3119 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val1.
getOperand(0));
3120 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3121 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.
getOperand(0),
3123 ReplaceUses(Ins1, NewIns);
3129 if (VT == MVT::v8i16 && Subtarget->hasFullFP16()) {
3130 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3131 ARM::ssub_0 + ExtractLane1 / 2, dl, MVT::f32, Val1.
getOperand(0));
3132 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3133 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val2.
getOperand(0));
3134 if (ExtractLane1 % 2 != 0)
3135 Inp1 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0);
3136 if (ExtractLane2 % 2 != 0)
3137 Inp2 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0);
3138 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1);
3140 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3142 ReplaceUses(Ins1, NewIns);
3149 if (VT == MVT::v8f16 && Subtarget->hasFullFP16()) {
3150 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1);
3152 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3154 ReplaceUses(Ins1, NewIns);
3161bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(
SDNode *
N,
3164 bool FixedToFloat) {
3165 auto Type =
N->getValueType(0);
3167 if (ScalarBits > 32)
3173 if (ScalarBits == 16 && !FMulFlags.
hasNoInfs() && IsUnsigned)
3198 if (!isa<ConstantSDNode>(ImmNode.
getOperand(0)))
3204 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3205 APInt(ScalarBits, Imm));
3222 if (!ImmAPF.getExactInverse(&ToConvert))
3225 APSInt Converted(64,
false);
3229 if (!IsExact || !Converted.isPowerOf2())
3232 unsigned FracBits = Converted.logBase2();
3233 if (FracBits > ScalarBits)
3237 VecVal, CurDAG->getConstant(FracBits,
SDLoc(
N), MVT::i32)};
3238 AddEmptyMVEPredicateToOps(Ops,
SDLoc(
N),
Type);
3240 unsigned int Opcode;
3241 switch (ScalarBits) {
3244 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3246 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3250 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3252 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3259 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
SDLoc(
N),
Type, Ops));
3263bool ARMDAGToDAGISel::tryFP_TO_INT(
SDNode *
N,
SDLoc dl) {
3265 if (!Subtarget->hasMVEFloatOps())
3268 if (!
Type.isVector())
3279 if (
Node->getOperand(0) !=
Node->getOperand(1))
3284 if (ScalarBits == 16 && !
Flags.hasNoInfs() && IsUnsigned)
3288 switch (ScalarBits) {
3290 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3293 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3297 CurDAG->getConstant(1, dl, MVT::i32)};
3298 AddEmptyMVEPredicateToOps(Ops, dl,
Type);
3300 ReplaceNode(
N, CurDAG->getMachineNode(Opcode, dl,
Type, Ops));
3307 return transformFixedFloatingPointConversion(
N,
Node, IsUnsigned,
false);
3310bool ARMDAGToDAGISel::tryFMULFixed(
SDNode *
N,
SDLoc dl) {
3312 if (!Subtarget->hasMVEFloatOps())
3314 auto Type =
N->getValueType(0);
3315 if (!
Type.isVector())
3318 auto LHS =
N->getOperand(0);
3322 return transformFixedFloatingPointConversion(
3326bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned) {
3327 if (!Subtarget->hasV6T2Ops())
3331 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3332 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3336 unsigned And_imm = 0;
3341 if (And_imm & (And_imm + 1))
3344 unsigned Srl_imm = 0;
3347 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3352 And_imm &= -1U >> Srl_imm;
3356 unsigned LSB = Srl_imm;
3358 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3360 if ((LSB + Width + 1) ==
N->getValueType(0).getSizeInBits()) {
3362 if (Subtarget->isThumb()) {
3363 Opc =
isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3364 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3365 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3366 getAL(CurDAG, dl), Reg0, Reg0 };
3367 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3376 SDValue Ops[] = {
N->getOperand(0).getOperand(0), ShOpc,
3377 getAL(CurDAG, dl), Reg0, Reg0 };
3378 CurDAG->SelectNodeTo(
N, ARM::MOVsi, MVT::i32, Ops);
3382 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3383 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3384 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3385 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3386 getAL(CurDAG, dl), Reg0 };
3387 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3395 unsigned Shl_imm = 0;
3397 assert(Shl_imm > 0 && Shl_imm < 32 &&
"bad amount in shift node!");
3398 unsigned Srl_imm = 0;
3400 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3402 unsigned Width = 32 - Srl_imm - 1;
3403 int LSB = Srl_imm - Shl_imm;
3406 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3407 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3408 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3409 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3410 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3411 getAL(CurDAG, dl), Reg0 };
3412 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3420 unsigned Srl_imm = 0;
3424 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3427 unsigned Width = MSB - LSB;
3428 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3429 assert(Srl_imm + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3430 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3431 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
3432 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3433 getAL(CurDAG, dl), Reg0 };
3434 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3440 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3446 if (LSB + Width > 32)
3449 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3450 assert(LSB + Width <= 32 &&
"Shouldn't create an invalid ubfx");
3451 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3452 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3453 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
3454 getAL(CurDAG, dl), Reg0 };
3455 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3472bool ARMDAGToDAGISel::tryABSOp(
SDNode *
N){
3473 SDValue SUBSrc0 =
N->getOperand(0);
3474 SDValue SUBSrc1 =
N->getOperand(1);
3475 EVT VT =
N->getValueType(0);
3491 if (XORSrc1 == SUBSrc1 && XORSrc0 == SRASrc0 && XType.
isInteger() &&
3493 unsigned Opcode = Subtarget->
isThumb2() ? ARM::t2ABS : ARM::ABS;
3494 CurDAG->SelectNodeTo(
N, Opcode, VT, XORSrc0);
3502void ARMDAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3504 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3505 if (MemTy == MVT::i8)
3506 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3507 else if (MemTy == MVT::i16)
3508 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3509 else if (MemTy == MVT::i32)
3510 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32;
3514 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3516 SDNode *CmpSwap = CurDAG->getMachineNode(
3518 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
3521 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3525 CurDAG->RemoveDeadNode(
N);
3528static std::optional<std::pair<unsigned, unsigned>>
3530 unsigned FirstOne =
A.getBitWidth() -
A.countl_zero() - 1;
3531 unsigned LastOne =
A.countr_zero();
3532 if (
A.popcount() != (FirstOne - LastOne + 1))
3533 return std::nullopt;
3534 return std::make_pair(FirstOne, LastOne);
3537void ARMDAGToDAGISel::SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI) {
3539 SwitchEQNEToPLMI =
false;
3541 if (!Subtarget->isThumb())
3548 if (!
And->hasOneUse())
3555 auto C = dyn_cast<ConstantSDNode>(
And.getOperand(1));
3569 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3570 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3571 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3572 CurDAG->getRegister(0, MVT::i32) };
3573 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3575 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
3576 CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3577 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3578 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3582 if (
Range->second == 0) {
3584 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3585 ReplaceNode(
And.getNode(), NewN);
3586 }
else if (
Range->first == 31) {
3588 NewN = EmitShift(ARM::tLSRri,
X,
Range->second);
3589 ReplaceNode(
And.getNode(), NewN);
3595 if (!
N->hasOneUse())
3597 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3598 ReplaceNode(
And.getNode(), NewN);
3600 SwitchEQNEToPLMI =
true;
3601 }
else if (!Subtarget->hasV6T2Ops()) {
3604 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3605 NewN = EmitShift(ARM::tLSRri,
SDValue(NewN, 0),
3607 ReplaceNode(
And.getNode(), NewN);
3612 unsigned Opc128[3]) {
3614 "Unexpected vector shuffle length");
3627void ARMDAGToDAGISel::Select(
SDNode *
N) {
3630 if (
N->isMachineOpcode()) {
3635 switch (
N->getOpcode()) {
3657 cast<RegisterSDNode>(
Ptr.getOperand(1))->getReg() == ARM::SP &&
3658 Ptr.getOperand(0) ==
ST->getChain()) {
3660 CurDAG->getRegister(ARM::SP, MVT::i32),
3661 CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
3663 CurDAG->getRegister(0, MVT::i32),
3666 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
3668 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3669 ReplaceNode(
N, ResNode);
3676 if (tryWriteRegister(
N))
3680 if (tryReadRegister(
N))
3685 if (tryInlineAsm(
N))
3695 unsigned Val =
N->getAsZExtVal();
3698 !Subtarget->genExecuteOnly()) {
3699 SDValue CPIdx = CurDAG->getTargetConstantPool(
3701 TLI->getPointerTy(CurDAG->getDataLayout()));
3704 if (Subtarget->isThumb()) {
3708 CurDAG->getRegister(0, MVT::i32),
3709 CurDAG->getEntryNode()
3711 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
3716 CurDAG->getTargetConstant(0, dl, MVT::i32),
3718 CurDAG->getRegister(0, MVT::i32),
3719 CurDAG->getEntryNode()
3721 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
3732 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3734 ReplaceNode(
N, ResNode);
3743 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
3744 SDValue TFI = CurDAG->getTargetFrameIndex(
3745 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3752 CurDAG->SelectNodeTo(
N, ARM::tADDframe, MVT::i32, TFI,
3753 CurDAG->getTargetConstant(0, dl, MVT::i32));
3756 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3757 ARM::t2ADDri : ARM::ADDri);
3758 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3759 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3760 CurDAG->getRegister(0, MVT::i32) };
3761 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3766 if (tryInsertVectorElt(
N))
3771 if (tryV6T2BitfieldExtractOp(
N,
false))
3776 if (tryV6T2BitfieldExtractOp(
N,
true))
3783 if (tryFP_TO_INT(
N, dl))
3787 if (tryFMULFixed(
N, dl))
3794 unsigned RHSV =
C->getZExtValue();
3797 unsigned ShImm =
Log2_32(RHSV-1);
3802 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3803 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3804 if (Subtarget->isThumb()) {
3805 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3806 CurDAG->SelectNodeTo(
N, ARM::t2ADDrs, MVT::i32, Ops);
3811 CurDAG->SelectNodeTo(
N, ARM::ADDrsi, MVT::i32, Ops);
3816 unsigned ShImm =
Log2_32(RHSV+1);
3821 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3822 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3823 if (Subtarget->isThumb()) {
3824 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3825 CurDAG->SelectNodeTo(
N, ARM::t2RSBrs, MVT::i32, Ops);
3830 CurDAG->SelectNodeTo(
N, ARM::RSBrsi, MVT::i32, Ops);
3838 if (tryV6T2BitfieldExtractOp(
N,
false))
3844 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3845 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3852 bool PreferImmediateEncoding =
3853 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3854 if (!PreferImmediateEncoding &&
3859 SDValue NewImm = CurDAG->getConstant(~Imm, dl, MVT::i32);
3863 CurDAG->RepositionNode(
N->getIterator(), NewImm.
getNode());
3865 if (!Subtarget->hasThumb2()) {
3866 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3867 N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3868 CurDAG->getRegister(0, MVT::i32)};
3869 ReplaceNode(
N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
3872 SDValue Ops[] = {
N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3873 CurDAG->getRegister(0, MVT::i32),
3874 CurDAG->getRegister(0, MVT::i32)};
3876 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
3887 EVT VT =
N->getValueType(0);
3890 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3892 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3895 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
3896 N1C = dyn_cast<ConstantSDNode>(N1);
3904 unsigned N1CVal = N1C->getZExtValue();
3906 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3907 (N1CVal & 0xffffU) == 0xffffU &&
3908 (N2CVal & 0xffffU) == 0x0U) {
3909 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3912 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
3913 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3921 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3922 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
3923 N->getOperand(2),
N->getOperand(3),
3925 CurDAG->getRegister(0, MVT::i32) };
3926 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
3930 if (Subtarget->isThumb()) {
3931 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3932 N->getOperand(3),
getAL(CurDAG, dl),
3933 CurDAG->getRegister(0, MVT::i32)};
3935 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
3938 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3939 N->getOperand(3),
getAL(CurDAG, dl),
3940 CurDAG->getRegister(0, MVT::i32),
3941 CurDAG->getRegister(0, MVT::i32) };
3942 ReplaceNode(
N, CurDAG->getMachineNode(
3943 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3944 MVT::i32, MVT::i32, Ops));
3949 if (Subtarget->isThumb()) {
3950 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3951 N->getOperand(3),
getAL(CurDAG, dl),
3952 CurDAG->getRegister(0, MVT::i32)};
3954 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
3957 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3958 N->getOperand(3),
getAL(CurDAG, dl),
3959 CurDAG->getRegister(0, MVT::i32),
3960 CurDAG->getRegister(0, MVT::i32) };
3961 ReplaceNode(
N, CurDAG->getMachineNode(
3962 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3963 MVT::i32, MVT::i32, Ops));
3968 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3977 if (Subtarget->isThumb())
3978 assert(Subtarget->hasThumb2() &&
3979 "This pattern should not be generated for Thumb");
3981 SDValue SmulLoHi =
N->getOperand(1);
3986 N->getOperand(1) != SmulLoHi.
getValue(1) ||
3990 unsigned Opc = Subtarget->
isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3992 N->getOperand(0),
getAL(CurDAG, dl),
3993 CurDAG->getRegister(0, MVT::i32) };
3994 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
3998 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4000 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
4001 if (tryT2IndexedLoad(
N))
4003 }
else if (Subtarget->isThumb()) {
4004 if (tryT1IndexedLoad(
N))
4006 }
else if (tryARMIndexedLoad(
N))
4012 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4017 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopSetup, dl, MVT::i32,
4019 ReplaceUses(
N, New);
4020 CurDAG->RemoveDeadNode(
N);
4024 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopStart, dl, MVT::Other,
4025 N->getOperand(1),
N->getOperand(2),
4027 ReplaceUses(
N, New);
4028 CurDAG->RemoveDeadNode(
N);
4032 SDValue Ops[] = {
N->getOperand(1),
4035 unsigned Opc = ARM::t2LoopEnd;
4036 SDNode *
New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
4037 ReplaceUses(
N, New);
4038 CurDAG->RemoveDeadNode(
N);
4045 const SDValue &Chain =
N->getOperand(0);
4047 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4048 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4054 RegOffset = CurDAG->getRegister(0, MVT::i32);
4056 SDValue Ops[] = {
Base, RegOffset, ImmOffset, Chain};
4057 SDNode *
New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4058 {MVT::Untyped, MVT::Other}, Ops);
4059 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4061 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4063 transferMemOperands(
N, New);
4067 CurDAG->RemoveDeadNode(
N);
4074 const SDValue &Chain =
N->getOperand(0);
4076 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4077 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4083 RegOffset = CurDAG->getRegister(0, MVT::i32);
4088 SDNode *
New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops);
4089 transferMemOperands(
N, New);
4091 CurDAG->RemoveDeadNode(
N);
4095 SDValue Ops[] = {
N->getOperand(1),
4099 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4100 CurDAG->getVTList(MVT::i32, MVT::Other), Ops);
4101 ReplaceUses(
N, Dec);
4102 CurDAG->RemoveDeadNode(
N);
4118 unsigned Opc = Subtarget->isThumb() ?
4119 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4135 if (
ID == Intrinsic::loop_decrement_reg) {
4137 SDValue Size = CurDAG->getTargetConstant(
Int.getConstantOperandVal(3),
4142 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4143 CurDAG->getVTList(MVT::i32, MVT::Other),
4145 ReplaceUses(
Int.getNode(), LoopDec);
4149 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
4151 ReplaceUses(
N, LoopEnd);
4152 CurDAG->RemoveDeadNode(
N);
4153 CurDAG->RemoveDeadNode(
Flags.getNode());
4154 CurDAG->RemoveDeadNode(
Int.getNode());
4159 bool SwitchEQNEToPLMI;
4160 SelectCMPZ(
Flags.getNode(), SwitchEQNEToPLMI);
4161 Flags =
N->getOperand(3);
4163 if (SwitchEQNEToPLMI) {
4176 SDValue Tmp2 = CurDAG->getTargetConstant(
CC, dl, MVT::i32);
4177 Chain = CurDAG->getCopyToReg(Chain, dl, ARM::CPSR, Flags,
SDValue());
4178 SDValue Ops[] = {N1, Tmp2, CurDAG->getRegister(ARM::CPSR, MVT::i32), Chain,
4180 CurDAG->SelectNodeTo(
N, Opc, MVT::Other, Ops);
4190 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1).getNode());
4191 if (
C &&
C->getSExtValue() < 0 && Subtarget->isThumb()) {
4192 int64_t Addend = -
C->getSExtValue();
4199 if (Addend < 1<<8) {
4201 SDValue Ops[] = {
X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4202 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
4203 CurDAG->getRegister(0, MVT::i32) };
4204 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
4206 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4207 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
X,
4208 CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4209 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
4210 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
4226 bool SwitchEQNEToPLMI;
4227 SelectCMPZ(
Flags.getNode(), SwitchEQNEToPLMI);
4229 if (SwitchEQNEToPLMI) {
4242 SDValue NewARMcc = CurDAG->getConstant((
unsigned)
CC, dl, MVT::i32);
4243 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), NewARMcc,
4252 EVT VT =
N->getValueType(0);
4254 unsigned Opc64[] = {ARM::VZIPd8, ARM::VZIPd16, ARM::VTRNd32};
4255 unsigned Opc128[] = {ARM::VZIPq8, ARM::VZIPq16, ARM::VZIPq32};
4258 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4259 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4260 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4264 EVT VT =
N->getValueType(0);
4266 unsigned Opc64[] = {ARM::VUZPd8, ARM::VUZPd16, ARM::VTRNd32};
4267 unsigned Opc128[] = {ARM::VUZPq8, ARM::VUZPq16, ARM::VUZPq32};
4270 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4271 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4272 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4276 EVT VT =
N->getValueType(0);
4277 unsigned Opc64[] = {ARM::VTRNd8, ARM::VTRNd16, ARM::VTRNd32};
4278 unsigned Opc128[] = {ARM::VTRNq8, ARM::VTRNq16, ARM::VTRNq32};
4281 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4282 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4283 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4287 EVT VecVT =
N->getValueType(0);
4290 if (EltVT == MVT::f64) {
4291 assert(NumElts == 2 &&
"unexpected type for BUILD_VECTOR");
4293 N, createDRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4296 assert(EltVT == MVT::f32 &&
"unexpected type for BUILD_VECTOR");
4299 N, createSRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4302 assert(NumElts == 4 &&
"unexpected type for BUILD_VECTOR");
4304 createQuadSRegsNode(VecVT,
N->getOperand(0),
N->getOperand(1),
4305 N->getOperand(2),
N->getOperand(3)));
4310 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4312 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4314 SelectVLDDup(
N,
false,
false, 1, DOpcodes, QOpcodes);
4319 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4321 SelectVLDDup(
N,
false,
false, 2, Opcodes);
4326 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4327 ARM::VLD3DUPd16Pseudo,
4328 ARM::VLD3DUPd32Pseudo };
4329 SelectVLDDup(
N,
false,
false, 3, Opcodes);
4334 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4335 ARM::VLD4DUPd16Pseudo,
4336 ARM::VLD4DUPd32Pseudo };
4337 SelectVLDDup(
N,
false,
false, 4, Opcodes);
4342 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4343 ARM::VLD1DUPd16wb_fixed,
4344 ARM::VLD1DUPd32wb_fixed };
4345 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4346 ARM::VLD1DUPq16wb_fixed,
4347 ARM::VLD1DUPq32wb_fixed };
4348 SelectVLDDup(
N,
false,
true, 1, DOpcodes, QOpcodes);
4353 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4354 ARM::VLD2DUPd16wb_fixed,
4355 ARM::VLD2DUPd32wb_fixed,
4356 ARM::VLD1q64wb_fixed };
4357 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4358 ARM::VLD2DUPq16EvenPseudo,
4359 ARM::VLD2DUPq32EvenPseudo };
4360 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4361 ARM::VLD2DUPq16OddPseudoWB_fixed,
4362 ARM::VLD2DUPq32OddPseudoWB_fixed };
4363 SelectVLDDup(
N,
false,
true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4368 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4369 ARM::VLD3DUPd16Pseudo_UPD,
4370 ARM::VLD3DUPd32Pseudo_UPD,
4371 ARM::VLD1d64TPseudoWB_fixed };
4372 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4373 ARM::VLD3DUPq16EvenPseudo,
4374 ARM::VLD3DUPq32EvenPseudo };
4375 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4376 ARM::VLD3DUPq16OddPseudo_UPD,
4377 ARM::VLD3DUPq32OddPseudo_UPD };
4378 SelectVLDDup(
N,
false,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4383 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4384 ARM::VLD4DUPd16Pseudo_UPD,
4385 ARM::VLD4DUPd32Pseudo_UPD,
4386 ARM::VLD1d64QPseudoWB_fixed };
4387 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4388 ARM::VLD4DUPq16EvenPseudo,
4389 ARM::VLD4DUPq32EvenPseudo };
4390 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4391 ARM::VLD4DUPq16OddPseudo_UPD,
4392 ARM::VLD4DUPq32OddPseudo_UPD };
4393 SelectVLDDup(
N,
false,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4398 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4399 ARM::VLD1d16wb_fixed,
4400 ARM::VLD1d32wb_fixed,
4401 ARM::VLD1d64wb_fixed };
4402 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4403 ARM::VLD1q16wb_fixed,
4404 ARM::VLD1q32wb_fixed,
4405 ARM::VLD1q64wb_fixed };
4406 SelectVLD(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4411 if (Subtarget->hasNEON()) {
4412 static const uint16_t DOpcodes[] = {
4413 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4414 ARM::VLD1q64wb_fixed};
4415 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4416 ARM::VLD2q16PseudoWB_fixed,
4417 ARM::VLD2q32PseudoWB_fixed};
4418 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4420 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4421 ARM::MVE_VLD21_8_wb};
4422 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4423 ARM::MVE_VLD21_16_wb};
4424 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4425 ARM::MVE_VLD21_32_wb};
4426 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4427 SelectMVE_VLD(
N, 2, Opcodes,
true);
4433 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4434 ARM::VLD3d16Pseudo_UPD,
4435 ARM::VLD3d32Pseudo_UPD,
4436 ARM::VLD1d64TPseudoWB_fixed};
4437 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4438 ARM::VLD3q16Pseudo_UPD,
4439 ARM::VLD3q32Pseudo_UPD };
4440 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4441 ARM::VLD3q16oddPseudo_UPD,
4442 ARM::VLD3q32oddPseudo_UPD };
4443 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4448 if (Subtarget->hasNEON()) {
4449 static const uint16_t DOpcodes[] = {
4450 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4451 ARM::VLD1d64QPseudoWB_fixed};
4452 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4453 ARM::VLD4q16Pseudo_UPD,
4454 ARM::VLD4q32Pseudo_UPD};
4455 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4456 ARM::VLD4q16oddPseudo_UPD,
4457 ARM::VLD4q32oddPseudo_UPD};
4458 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4460 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4462 ARM::MVE_VLD43_8_wb};
4463 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4465 ARM::MVE_VLD43_16_wb};
4466 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4468 ARM::MVE_VLD43_32_wb};
4469 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4470 SelectMVE_VLD(
N, 4, Opcodes,
true);
4476 if (Subtarget->hasNEON()) {
4477 static const uint16_t DOpcodes[] = {
4478 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4479 ARM::VLD1q64wb_fixed};
4480 static const uint16_t QOpcodes[] = {
4481 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4482 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4483 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4490 if (Subtarget->hasNEON()) {
4491 static const uint16_t DOpcodes[] = {
4492 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4493 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4494 static const uint16_t QOpcodes0[] = {
4495 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4496 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4497 static const uint16_t QOpcodes1[] = {
4498 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4499 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4500 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4507 if (Subtarget->hasNEON()) {
4508 static const uint16_t DOpcodes[] = {
4509 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4510 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4511 static const uint16_t QOpcodes0[] = {
4512 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4513 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4514 static const uint16_t QOpcodes1[] = {
4515 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4516 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4517 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4524 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4525 ARM::VLD2LNd16Pseudo_UPD,
4526 ARM::VLD2LNd32Pseudo_UPD };
4527 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4528 ARM::VLD2LNq32Pseudo_UPD };
4529 SelectVLDSTLane(
N,
true,
true, 2, DOpcodes, QOpcodes);
4534 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4535 ARM::VLD3LNd16Pseudo_UPD,
4536 ARM::VLD3LNd32Pseudo_UPD };
4537 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4538 ARM::VLD3LNq32Pseudo_UPD };
4539 SelectVLDSTLane(
N,
true,
true, 3, DOpcodes, QOpcodes);
4544 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4545 ARM::VLD4LNd16Pseudo_UPD,
4546 ARM::VLD4LNd32Pseudo_UPD };
4547 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4548 ARM::VLD4LNq32Pseudo_UPD };
4549 SelectVLDSTLane(
N,
true,
true, 4, DOpcodes, QOpcodes);
4554 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4555 ARM::VST1d16wb_fixed,
4556 ARM::VST1d32wb_fixed,
4557 ARM::VST1d64wb_fixed };
4558 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4559 ARM::VST1q16wb_fixed,
4560 ARM::VST1q32wb_fixed,
4561 ARM::VST1q64wb_fixed };
4562 SelectVST(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4567 if (Subtarget->hasNEON()) {
4568 static const uint16_t DOpcodes[] = {
4569 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4570 ARM::VST1q64wb_fixed};
4571 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4572 ARM::VST2q16PseudoWB_fixed,
4573 ARM::VST2q32PseudoWB_fixed};
4574 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4581 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4582 ARM::VST3d16Pseudo_UPD,
4583 ARM::VST3d32Pseudo_UPD,
4584 ARM::VST1d64TPseudoWB_fixed};
4585 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4586 ARM::VST3q16Pseudo_UPD,
4587 ARM::VST3q32Pseudo_UPD };
4588 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4589 ARM::VST3q16oddPseudo_UPD,
4590 ARM::VST3q32oddPseudo_UPD };
4591 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4596 if (Subtarget->hasNEON()) {
4597 static const uint16_t DOpcodes[] = {
4598 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4599 ARM::VST1d64QPseudoWB_fixed};
4600 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4601 ARM::VST4q16Pseudo_UPD,
4602 ARM::VST4q32Pseudo_UPD};
4603 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4604 ARM::VST4q16oddPseudo_UPD,
4605 ARM::VST4q32oddPseudo_UPD};
4606 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4613 if (Subtarget->hasNEON()) {
4614 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4615 ARM::VST1q16wb_fixed,
4616 ARM::VST1q32wb_fixed,
4617 ARM::VST1q64wb_fixed};
4618 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4619 ARM::VST1d16QPseudoWB_fixed,
4620 ARM::VST1d32QPseudoWB_fixed,
4621 ARM::VST1d64QPseudoWB_fixed };
4622 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4629 if (Subtarget->hasNEON()) {
4630 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4631 ARM::VST1d16TPseudoWB_fixed,
4632 ARM::VST1d32TPseudoWB_fixed,
4633 ARM::VST1d64TPseudoWB_fixed };
4634 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4635 ARM::VST1q16LowTPseudo_UPD,
4636 ARM::VST1q32LowTPseudo_UPD,
4637 ARM::VST1q64LowTPseudo_UPD };
4638 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4639 ARM::VST1q16HighTPseudo_UPD,
4640 ARM::VST1q32HighTPseudo_UPD,
4641 ARM::VST1q64HighTPseudo_UPD };
4642 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4649 if (Subtarget->hasNEON()) {
4650 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4651 ARM::VST1d16QPseudoWB_fixed,
4652 ARM::VST1d32QPseudoWB_fixed,
4653 ARM::VST1d64QPseudoWB_fixed };
4654 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4655 ARM::VST1q16LowQPseudo_UPD,
4656 ARM::VST1q32LowQPseudo_UPD,
4657 ARM::VST1q64LowQPseudo_UPD };
4658 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4659 ARM::VST1q16HighQPseudo_UPD,
4660 ARM::VST1q32HighQPseudo_UPD,
4661 ARM::VST1q64HighQPseudo_UPD };
4662 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4668 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4669 ARM::VST2LNd16Pseudo_UPD,
4670 ARM::VST2LNd32Pseudo_UPD };
4671 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4672 ARM::VST2LNq32Pseudo_UPD };
4673 SelectVLDSTLane(
N,
false,
true, 2, DOpcodes, QOpcodes);
4678 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4679 ARM::VST3LNd16Pseudo_UPD,
4680 ARM::VST3LNd32Pseudo_UPD };
4681 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4682 ARM::VST3LNq32Pseudo_UPD };
4683 SelectVLDSTLane(
N,
false,
true, 3, DOpcodes, QOpcodes);
4688 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4689 ARM::VST4LNd16Pseudo_UPD,
4690 ARM::VST4LNd32Pseudo_UPD };
4691 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4692 ARM::VST4LNq32Pseudo_UPD };
4693 SelectVLDSTLane(
N,
false,
true, 4, DOpcodes, QOpcodes);
4699 unsigned IntNo =
N->getConstantOperandVal(1);
4704 case Intrinsic::arm_mrrc:
4705 case Intrinsic::arm_mrrc2: {
4710 if (Subtarget->isThumb())
4711 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4713 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4716 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(2), dl));
4717 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(3), dl));
4718 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(4), dl));
4723 if (Opc != ARM::MRRC2) {
4725 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4731 const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
4733 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4736 case Intrinsic::arm_ldaexd:
4737 case Intrinsic::arm_ldrexd: {
4740 SDValue MemAddr =
N->getOperand(2);
4741 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4743 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4744 unsigned NewOpc =
isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4745 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4748 std::vector<EVT> ResTys;
4750 ResTys.push_back(MVT::i32);
4751 ResTys.push_back(MVT::i32);
4753 ResTys.push_back(MVT::Untyped);
4754 ResTys.push_back(MVT::Other);
4758 CurDAG->getRegister(0, MVT::i32), Chain};
4759 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4762 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4772 CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
4773 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4774 dl, MVT::i32,
SDValue(Ld, 0), SubRegIdx);
4777 ReplaceUses(
SDValue(
N, 0), Result);
4785 CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
4786 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4787 dl, MVT::i32,
SDValue(Ld, 0), SubRegIdx);
4790 ReplaceUses(
SDValue(
N, 1), Result);
4792 ReplaceUses(
SDValue(
N, 2), OutChain);
4793 CurDAG->RemoveDeadNode(
N);
4796 case Intrinsic::arm_stlexd:
4797 case Intrinsic::arm_strexd: {
4802 SDValue MemAddr =
N->getOperand(4);
4806 const EVT ResTys[] = {MVT::i32, MVT::Other};
4808 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
4819 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4822 bool IsRelease = IntNo == Intrinsic::arm_stlexd;
4823 unsigned NewOpc =
isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
4824 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
4826 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4829 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4835 case Intrinsic::arm_neon_vld1: {
4836 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
4837 ARM::VLD1d32, ARM::VLD1d64 };
4838 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4839 ARM::VLD1q32, ARM::VLD1q64};
4840 SelectVLD(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
4844 case Intrinsic::arm_neon_vld1x2: {
4845 static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4846 ARM::VLD1q32, ARM::VLD1q64 };
4847 static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
4848 ARM::VLD1d16QPseudo,
4849 ARM::VLD1d32QPseudo,
4850 ARM::VLD1d64QPseudo };
4851 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4855 case Intrinsic::arm_neon_vld1x3: {
4856 static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
4857 ARM::VLD1d16TPseudo,
4858 ARM::VLD1d32TPseudo,
4859 ARM::VLD1d64TPseudo };
4860 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
4861 ARM::VLD1q16LowTPseudo_UPD,
4862 ARM::VLD1q32LowTPseudo_UPD,
4863 ARM::VLD1q64LowTPseudo_UPD };
4864 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
4865 ARM::VLD1q16HighTPseudo,
4866 ARM::VLD1q32HighTPseudo,
4867 ARM::VLD1q64HighTPseudo };
4868 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4872 case Intrinsic::arm_neon_vld1x4: {
4873 static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
4874 ARM::VLD1d16QPseudo,
4875 ARM::VLD1d32QPseudo,
4876 ARM::VLD1d64QPseudo };
4877 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
4878 ARM::VLD1q16LowQPseudo_UPD,
4879 ARM::VLD1q32LowQPseudo_UPD,
4880 ARM::VLD1q64LowQPseudo_UPD };
4881 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
4882 ARM::VLD1q16HighQPseudo,
4883 ARM::VLD1q32HighQPseudo,
4884 ARM::VLD1q64HighQPseudo };
4885 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4889 case Intrinsic::arm_neon_vld2: {
4890 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
4891 ARM::VLD2d32, ARM::VLD1q64 };
4892 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
4893 ARM::VLD2q32Pseudo };
4894 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4898 case Intrinsic::arm_neon_vld3: {
4899 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
4902 ARM::VLD1d64TPseudo };
4903 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4904 ARM::VLD3q16Pseudo_UPD,
4905 ARM::VLD3q32Pseudo_UPD };
4906 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
4907 ARM::VLD3q16oddPseudo,
4908 ARM::VLD3q32oddPseudo };
4909 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4913 case Intrinsic::arm_neon_vld4: {
4914 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
4917 ARM::VLD1d64QPseudo };
4918 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
4919 ARM::VLD4q16Pseudo_UPD,
4920 ARM::VLD4q32Pseudo_UPD };
4921 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
4922 ARM::VLD4q16oddPseudo,
4923 ARM::VLD4q32oddPseudo };
4924 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4928 case Intrinsic::arm_neon_vld2dup: {
4929 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4930 ARM::VLD2DUPd32, ARM::VLD1q64 };
4931 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4932 ARM::VLD2DUPq16EvenPseudo,
4933 ARM::VLD2DUPq32EvenPseudo };
4934 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
4935 ARM::VLD2DUPq16OddPseudo,
4936 ARM::VLD2DUPq32OddPseudo };
4937 SelectVLDDup(
N,
true,
false, 2,
4938 DOpcodes, QOpcodes0, QOpcodes1);
4942 case Intrinsic::arm_neon_vld3dup: {
4943 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
4944 ARM::VLD3DUPd16Pseudo,
4945 ARM::VLD3DUPd32Pseudo,
4946 ARM::VLD1d64TPseudo };
4947 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4948 ARM::VLD3DUPq16EvenPseudo,
4949 ARM::VLD3DUPq32EvenPseudo };
4950 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
4951 ARM::VLD3DUPq16OddPseudo,
4952 ARM::VLD3DUPq32OddPseudo };
4953 SelectVLDDup(
N,
true,
false, 3,
4954 DOpcodes, QOpcodes0, QOpcodes1);
4958 case Intrinsic::arm_neon_vld4dup: {
4959 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
4960 ARM::VLD4DUPd16Pseudo,
4961 ARM::VLD4DUPd32Pseudo,
4962 ARM::VLD1d64QPseudo };
4963 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4964 ARM::VLD4DUPq16EvenPseudo,
4965 ARM::VLD4DUPq32EvenPseudo };
4966 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
4967 ARM::VLD4DUPq16OddPseudo,
4968 ARM::VLD4DUPq32OddPseudo };
4969 SelectVLDDup(
N,
true,
false, 4,
4970 DOpcodes, QOpcodes0, QOpcodes1);
4974 case Intrinsic::arm_neon_vld2lane: {
4975 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
4976 ARM::VLD2LNd16Pseudo,
4977 ARM::VLD2LNd32Pseudo };
4978 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
4979 ARM::VLD2LNq32Pseudo };
4980 SelectVLDSTLane(
N,
true,
false, 2, DOpcodes, QOpcodes);
4984 case Intrinsic::arm_neon_vld3lane: {
4985 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
4986 ARM::VLD3LNd16Pseudo,
4987 ARM::VLD3LNd32Pseudo };
4988 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
4989 ARM::VLD3LNq32Pseudo };
4990 SelectVLDSTLane(
N,
true,
false, 3, DOpcodes, QOpcodes);
4994 case Intrinsic::arm_neon_vld4lane: {
4995 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
4996 ARM::VLD4LNd16Pseudo,
4997 ARM::VLD4LNd32Pseudo };
4998 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
4999 ARM::VLD4LNq32Pseudo };
5000 SelectVLDSTLane(
N,
true,
false, 4, DOpcodes, QOpcodes);
5004 case Intrinsic::arm_neon_vst1: {
5005 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
5006 ARM::VST1d32, ARM::VST1d64 };
5007 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5008 ARM::VST1q32, ARM::VST1q64 };
5009 SelectVST(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
5013 case Intrinsic::arm_neon_vst1x2: {
5014 static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5015 ARM::VST1q32, ARM::VST1q64 };
5016 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
5017 ARM::VST1d16QPseudo,
5018 ARM::VST1d32QPseudo,
5019 ARM::VST1d64QPseudo };
5020 SelectVST(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
5024 case Intrinsic::arm_neon_vst1x3: {
5025 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
5026 ARM::VST1d16TPseudo,
5027 ARM::VST1d32TPseudo,
5028 ARM::VST1d64TPseudo };
5029 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
5030 ARM::VST1q16LowTPseudo_UPD,
5031 ARM::VST1q32LowTPseudo_UPD,
5032 ARM::VST1q64LowTPseudo_UPD };
5033 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
5034 ARM::VST1q16HighTPseudo,
5035 ARM::VST1q32HighTPseudo,
5036 ARM::VST1q64HighTPseudo };
5037 SelectVST(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5041 case Intrinsic::arm_neon_vst1x4: {
5042 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
5043 ARM::VST1d16QPseudo,
5044 ARM::VST1d32QPseudo,
5045 ARM::VST1d64QPseudo };
5046 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
5047 ARM::VST1q16LowQPseudo_UPD,
5048 ARM::VST1q32LowQPseudo_UPD,
5049 ARM::VST1q64LowQPseudo_UPD };
5050 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
5051 ARM::VST1q16HighQPseudo,
5052 ARM::VST1q32HighQPseudo,
5053 ARM::VST1q64HighQPseudo };
5054 SelectVST(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5058 case Intrinsic::arm_neon_vst2: {
5059 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
5060 ARM::VST2d32, ARM::VST1q64 };
5061 static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
5062 ARM::VST2q32Pseudo };
5063 SelectVST(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
5067 case Intrinsic::arm_neon_vst3: {
5068 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
5071 ARM::VST1d64TPseudo };
5072 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
5073 ARM::VST3q16Pseudo_UPD,
5074 ARM::VST3q32Pseudo_UPD };
5075 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
5076 ARM::VST3q16oddPseudo,
5077 ARM::VST3q32oddPseudo };
5078 SelectVST(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5082 case Intrinsic::arm_neon_vst4: {
5083 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
5086 ARM::VST1d64QPseudo };
5087 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
5088 ARM::VST4q16Pseudo_UPD,
5089 ARM::VST4q32Pseudo_UPD };
5090 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
5091 ARM::VST4q16oddPseudo,
5092 ARM::VST4q32oddPseudo };
5093 SelectVST(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5097 case Intrinsic::arm_neon_vst2lane: {
5098 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
5099 ARM::VST2LNd16Pseudo,
5100 ARM::VST2LNd32Pseudo };
5101 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
5102 ARM::VST2LNq32Pseudo };
5103 SelectVLDSTLane(
N,
false,
false, 2, DOpcodes, QOpcodes);
5107 case Intrinsic::arm_neon_vst3lane: {
5108 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
5109 ARM::VST3LNd16Pseudo,
5110 ARM::VST3LNd32Pseudo };
5111 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
5112 ARM::VST3LNq32Pseudo };
5113 SelectVLDSTLane(
N,
false,
false, 3, DOpcodes, QOpcodes);
5117 case Intrinsic::arm_neon_vst4lane: {
5118 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
5119 ARM::VST4LNd16Pseudo,
5120 ARM::VST4LNd32Pseudo };
5121 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
5122 ARM::VST4LNq32Pseudo };
5123 SelectVLDSTLane(
N,
false,
false, 4, DOpcodes, QOpcodes);
5127 case Intrinsic::arm_mve_vldr_gather_base_wb:
5128 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
5129 static const uint16_t Opcodes[] = {ARM::MVE_VLDRWU32_qi_pre,
5130 ARM::MVE_VLDRDU64_qi_pre};
5131 SelectMVE_WB(
N, Opcodes,
5132 IntNo == Intrinsic::arm_mve_vldr_gather_base_wb_predicated);
5136 case Intrinsic::arm_mve_vld2q: {
5137 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8, ARM::MVE_VLD21_8};
5138 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
5140 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
5142 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5143 SelectMVE_VLD(
N, 2, Opcodes,
false);
5147 case Intrinsic::arm_mve_vld4q: {
5148 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
5149 ARM::MVE_VLD42_8, ARM::MVE_VLD43_8};
5150 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
5153 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
5156 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5157 SelectMVE_VLD(
N, 4, Opcodes,
false);
5165 unsigned IntNo =
N->getConstantOperandVal(0);
5171 case Intrinsic::arm_neon_vcvtbfp2bf: {
5173 const SDValue &Src =
N->getOperand(1);
5176 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5177 SDValue Ops[] = { Src, Src, Pred, Reg0 };
5178 CurDAG->SelectNodeTo(
N, ARM::BF16_VCVTB, DestTy, Ops);
5183 case Intrinsic::arm_neon_vcvtfp2bf: {
5185 const SDValue &Src =
N->getOperand(1);
5187 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5188 SDValue Ops[] = { Src, Pred, Reg0 };
5189 CurDAG->SelectNodeTo(
N, ARM::BF16_VCVT, MVT::v4bf16, Ops);
5193 case Intrinsic::arm_mve_urshrl:
5194 SelectMVE_LongShift(
N, ARM::MVE_URSHRL,
true,
false);
5196 case Intrinsic::arm_mve_uqshll:
5197 SelectMVE_LongShift(
N, ARM::MVE_UQSHLL,
true,
false);
5199 case Intrinsic::arm_mve_srshrl:
5200 SelectMVE_LongShift(
N, ARM::MVE_SRSHRL,
true,
false);
5202 case Intrinsic::arm_mve_sqshll:
5203 SelectMVE_LongShift(
N, ARM::MVE_SQSHLL,
true,
false);
5205 case Intrinsic::arm_mve_uqrshll:
5206 SelectMVE_LongShift(
N, ARM::MVE_UQRSHLL,
false,
true);
5208 case Intrinsic::arm_mve_sqrshrl:
5209 SelectMVE_LongShift(
N, ARM::MVE_SQRSHRL,
false,
true);
5212 case Intrinsic::arm_mve_vadc:
5213 case Intrinsic::arm_mve_vadc_predicated:
5214 SelectMVE_VADCSBC(
N, ARM::MVE_VADC, ARM::MVE_VADCI,
true,
5215 IntNo == Intrinsic::arm_mve_vadc_predicated);
5217 case Intrinsic::arm_mve_vsbc:
5218 case Intrinsic::arm_mve_vsbc_predicated:
5219 SelectMVE_VADCSBC(
N, ARM::MVE_VSBC, ARM::MVE_VSBCI,
false,
5220 IntNo == Intrinsic::arm_mve_vsbc_predicated);
5222 case Intrinsic::arm_mve_vshlc:
5223 case Intrinsic::arm_mve_vshlc_predicated:
5224 SelectMVE_VSHLC(
N, IntNo == Intrinsic::arm_mve_vshlc_predicated);
5227 case Intrinsic::arm_mve_vmlldava:
5228 case Intrinsic::arm_mve_vmlldava_predicated: {
5229 static const uint16_t OpcodesU[] = {
5230 ARM::MVE_VMLALDAVu16, ARM::MVE_VMLALDAVu32,
5231 ARM::MVE_VMLALDAVau16, ARM::MVE_VMLALDAVau32,
5233 static const uint16_t OpcodesS[] = {
5234 ARM::MVE_VMLALDAVs16, ARM::MVE_VMLALDAVs32,
5235 ARM::MVE_VMLALDAVas16, ARM::MVE_VMLALDAVas32,
5236 ARM::MVE_VMLALDAVxs16, ARM::MVE_VMLALDAVxs32,
5237 ARM::MVE_VMLALDAVaxs16, ARM::MVE_VMLALDAVaxs32,
5238 ARM::MVE_VMLSLDAVs16, ARM::MVE_VMLSLDAVs32,
5239 ARM::MVE_VMLSLDAVas16, ARM::MVE_VMLSLDAVas32,
5240 ARM::MVE_VMLSLDAVxs16, ARM::MVE_VMLSLDAVxs32,
5241 ARM::MVE_VMLSLDAVaxs16, ARM::MVE_VMLSLDAVaxs32,
5243 SelectMVE_VMLLDAV(
N, IntNo == Intrinsic::arm_mve_vmlldava_predicated,
5244 OpcodesS, OpcodesU);
5248 case Intrinsic::arm_mve_vrmlldavha:
5249 case Intrinsic::arm_mve_vrmlldavha_predicated: {
5250 static const uint16_t OpcodesU[] = {
5251 ARM::MVE_VRMLALDAVHu32, ARM::MVE_VRMLALDAVHau32,
5253 static const uint16_t OpcodesS[] = {
5254 ARM::MVE_VRMLALDAVHs32, ARM::MVE_VRMLALDAVHas32,
5255 ARM::MVE_VRMLALDAVHxs32, ARM::MVE_VRMLALDAVHaxs32,
5256 ARM::MVE_VRMLSLDAVHs32, ARM::MVE_VRMLSLDAVHas32,
5257 ARM::MVE_VRMLSLDAVHxs32, ARM::MVE_VRMLSLDAVHaxs32,
5259 SelectMVE_VRMLLDAVH(
N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated,
5260 OpcodesS, OpcodesU);
5264 case Intrinsic::arm_mve_vidup:
5265 case Intrinsic::arm_mve_vidup_predicated: {
5266 static const uint16_t Opcodes[] = {
5267 ARM::MVE_VIDUPu8, ARM::MVE_VIDUPu16, ARM::MVE_VIDUPu32,
5269 SelectMVE_VxDUP(
N, Opcodes,
false,
5270 IntNo == Intrinsic::arm_mve_vidup_predicated);
5274 case Intrinsic::arm_mve_vddup:
5275 case Intrinsic::arm_mve_vddup_predicated: {
5276 static const uint16_t Opcodes[] = {
5277 ARM::MVE_VDDUPu8, ARM::MVE_VDDUPu16, ARM::MVE_VDDUPu32,
5279 SelectMVE_VxDUP(
N, Opcodes,
false,
5280 IntNo == Intrinsic::arm_mve_vddup_predicated);
5284 case Intrinsic::arm_mve_viwdup:
5285 case Intrinsic::arm_mve_viwdup_predicated: {
5286 static const uint16_t Opcodes[] = {
5287 ARM::MVE_VIWDUPu8, ARM::MVE_VIWDUPu16, ARM::MVE_VIWDUPu32,
5289 SelectMVE_VxDUP(
N, Opcodes,
true,
5290 IntNo == Intrinsic::arm_mve_viwdup_predicated);
5294 case Intrinsic::arm_mve_vdwdup:
5295 case Intrinsic::arm_mve_vdwdup_predicated: {
5296 static const uint16_t Opcodes[] = {
5297 ARM::MVE_VDWDUPu8, ARM::MVE_VDWDUPu16, ARM::MVE_VDWDUPu32,
5299 SelectMVE_VxDUP(
N, Opcodes,
true,
5300 IntNo == Intrinsic::arm_mve_vdwdup_predicated);
5304 case Intrinsic::arm_cde_cx1d:
5305 case Intrinsic::arm_cde_cx1da:
5306 case Intrinsic::arm_cde_cx2d:
5307 case Intrinsic::arm_cde_cx2da:
5308 case Intrinsic::arm_cde_cx3d:
5309 case Intrinsic::arm_cde_cx3da: {
5310 bool HasAccum = IntNo == Intrinsic::arm_cde_cx1da ||
5311 IntNo == Intrinsic::arm_cde_cx2da ||
5312 IntNo == Intrinsic::arm_cde_cx3da;
5316 case Intrinsic::arm_cde_cx1d:
5317 case Intrinsic::arm_cde_cx1da:
5319 Opcode = HasAccum ? ARM::CDE_CX1DA : ARM::CDE_CX1D;
5321 case Intrinsic::arm_cde_cx2d:
5322 case Intrinsic::arm_cde_cx2da:
5324 Opcode = HasAccum ? ARM::CDE_CX2DA : ARM::CDE_CX2D;
5326 case Intrinsic::arm_cde_cx3d:
5327 case Intrinsic::arm_cde_cx3da:
5329 Opcode = HasAccum ? ARM::CDE_CX3DA : ARM::CDE_CX3D;
5334 SelectCDE_CXxD(
N, Opcode, NumExtraOps, HasAccum);
5357 std::vector<SDValue> &Ops) {
5359 RegString.
split(Fields,
':');
5361 if (Fields.
size() > 1) {
5362 bool AllIntFields =
true;
5367 AllIntFields &= !
Field.trim(
"CPcp").getAsInteger(10, IntField);
5372 "Unexpected non-integer value in special register string.");
5383 auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.
lower());
5386 return TheReg->Encoding;
5398 .
Case(
"nzcvqg", 0x3)
5406 auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
5407 const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
5408 if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
5410 return (
int)(TheReg->Encoding & 0xFFF);
5419 if (Reg ==
"apsr") {
5429 if (Reg !=
"cpsr" && Reg !=
"spsr") {
5434 if (Flags.empty() || Flags ==
"all")
5439 for (
char Flag : Flags) {
5459 if (!FlagVal || (Mask & FlagVal))
5474bool ARMDAGToDAGISel::tryReadRegister(
SDNode *
N){
5475 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
5476 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5477 bool IsThumb2 = Subtarget->
isThumb2();
5480 std::vector<SDValue> Ops;
5490 if (Ops.size() == 5){
5491 Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
5492 ResTypes.
append({ MVT::i32, MVT::Other });
5494 assert(Ops.size() == 3 &&
5495 "Invalid number of fields in special register string.");
5496 Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
5497 ResTypes.
append({ MVT::i32, MVT::i32, MVT::Other });
5500 Ops.push_back(
getAL(CurDAG,
DL));
5501 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5502 Ops.push_back(
N->getOperand(0));
5503 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, ResTypes, Ops));
5507 std::string SpecialReg = RegString->getString().lower();
5510 if (BankedReg != -1) {
5511 Ops = { CurDAG->getTargetConstant(BankedReg,
DL, MVT::i32),
5512 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5515 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
5516 DL, MVT::i32, MVT::Other, Ops));
5524 .
Case(
"fpscr", ARM::VMRS)
5525 .
Case(
"fpexc", ARM::VMRS_FPEXC)
5526 .
Case(
"fpsid", ARM::VMRS_FPSID)
5527 .
Case(
"mvfr0", ARM::VMRS_MVFR0)
5528 .
Case(
"mvfr1", ARM::VMRS_MVFR1)
5529 .
Case(
"mvfr2", ARM::VMRS_MVFR2)
5530 .
Case(
"fpinst", ARM::VMRS_FPINST)
5531 .
Case(
"fpinst2", ARM::VMRS_FPINST2)
5541 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5544 CurDAG->getMachineNode(Opcode,
DL, MVT::i32, MVT::Other, Ops));
5553 if (SYSmValue == -1)
5556 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue,
DL, MVT::i32),
5557 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5560 N, CurDAG->getMachineNode(ARM::t2MRS_M,
DL, MVT::i32, MVT::Other, Ops));
5566 if (SpecialReg ==
"apsr" || SpecialReg ==
"cpsr") {
5567 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5569 ReplaceNode(
N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
5570 DL, MVT::i32, MVT::Other, Ops));
5574 if (SpecialReg ==
"spsr") {
5575 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5578 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys,
DL,
5579 MVT::i32, MVT::Other, Ops));
5589bool ARMDAGToDAGISel::tryWriteRegister(
SDNode *
N){
5590 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
5591 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5592 bool IsThumb2 = Subtarget->
isThumb2();
5595 std::vector<SDValue> Ops;
5604 if (Ops.size() == 5) {
5605 Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
5606 Ops.insert(Ops.begin()+2,
N->getOperand(2));
5608 assert(Ops.size() == 3 &&
5609 "Invalid number of fields in special register string.");
5610 Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
5611 SDValue WriteValue[] = {
N->getOperand(2),
N->getOperand(3) };
5612 Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
5615 Ops.push_back(
getAL(CurDAG,
DL));
5616 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5617 Ops.push_back(
N->getOperand(0));
5619 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
5623 std::string SpecialReg = RegString->getString().lower();
5625 if (BankedReg != -1) {
5626 Ops = { CurDAG->getTargetConstant(BankedReg,
DL, MVT::i32),
N->getOperand(2),
5627 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5630 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
5631 DL, MVT::Other, Ops));
5639 .
Case(
"fpscr", ARM::VMSR)
5640 .
Case(
"fpexc", ARM::VMSR_FPEXC)
5641 .
Case(
"fpsid", ARM::VMSR_FPSID)
5642 .
Case(
"fpinst", ARM::VMSR_FPINST)
5643 .
Case(
"fpinst2", ARM::VMSR_FPINST2)
5649 Ops = {
N->getOperand(2),
getAL(CurDAG,
DL),
5650 CurDAG->getRegister(0, MVT::i32),
N->getOperand(0) };
5651 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
5655 std::pair<StringRef, StringRef> Fields;
5657 std::string
Reg = Fields.first.str();
5664 if (SYSmValue == -1)
5667 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue,
DL, MVT::i32),
5668 N->getOperand(2),
getAL(CurDAG,
DL),
5669 CurDAG->getRegister(0, MVT::i32),
N->getOperand(0) };
5670 ReplaceNode(
N, CurDAG->getMachineNode(ARM::t2MSR_M,
DL, MVT::Other, Ops));
5679 Ops = { CurDAG->getTargetConstant(Mask,
DL, MVT::i32),
N->getOperand(2),
5680 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5682 ReplaceNode(
N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
5683 DL, MVT::Other, Ops));
5690bool ARMDAGToDAGISel::tryInlineAsm(
SDNode *
N){
5691 std::vector<SDValue> AsmNodeOperands;
5693 bool Changed =
false;
5694 unsigned NumOps =
N->getNumOperands();
5709 for(
unsigned i = 0, e =
N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
5711 AsmNodeOperands.push_back(
op);
5716 if (
const auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(i)))
5725 if (
Flag.isImmKind()) {
5727 AsmNodeOperands.push_back(
op);
5731 const unsigned NumRegs =
Flag.getNumOperandRegisters();
5735 unsigned DefIdx = 0;
5736 bool IsTiedToChangedOp =
false;
5739 if (Changed &&
Flag.isUseOperandTiedToDef(DefIdx))
5740 IsTiedToChangedOp = OpChanged[DefIdx];
5747 if (
Flag.isMemKind()) {
5749 AsmNodeOperands.push_back(
op);
5753 if (!
Flag.isRegUseKind() && !
Flag.isRegDefKind() &&
5754 !
Flag.isRegDefEarlyClobberKind())
5758 const bool HasRC =
Flag.hasRegClassConstraint(RC);
5759 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
5763 assert((i+2 < NumOps) &&
"Invalid number of operands in inline asm");
5766 Register Reg0 = cast<RegisterSDNode>(V0)->getReg();
5767 Register Reg1 = cast<RegisterSDNode>(V1)->getReg();
5771 if (
Flag.isRegDefKind() ||
Flag.isRegDefEarlyClobberKind()) {
5775 Register GPVR =
MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5776 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5779 SDNode *GU =
N->getGluedUser();
5780 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
5784 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
5786 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
5788 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
5794 Ops.push_back(
T1.getValue(1));
5795 CurDAG->UpdateNodeOperands(GU, Ops);
5802 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
5804 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
5810 Register GPVR =
MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5811 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5812 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair,
T1.getValue(1));
5821 OpChanged[OpChanged.
size() -1 ] =
true;
5823 if (IsTiedToChangedOp)
5824 Flag.setMatchingOp(DefIdx);
5826 Flag.setRegClass(ARM::GPRPairRegClassID);
5828 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
5829 Flag, dl, MVT::i32);
5831 AsmNodeOperands.push_back(PairedReg);
5838 AsmNodeOperands.push_back(Glue);
5843 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
5845 ReplaceNode(
N,
New.getNode());
5849bool ARMDAGToDAGISel::SelectInlineAsmMemoryOperand(
5851 std::vector<SDValue> &OutOps) {
5852 switch(ConstraintID) {
5855 case InlineAsm::ConstraintCode::m:
5856 case InlineAsm::ConstraintCode::o:
5857 case InlineAsm::ConstraintCode::Q:
5858 case InlineAsm::ConstraintCode::Um:
5859 case InlineAsm::ConstraintCode::Un:
5860 case InlineAsm::ConstraintCode::Uq:
5861 case InlineAsm::ConstraintCode::Us:
5862 case InlineAsm::ConstraintCode::Ut:
5863 case InlineAsm::ConstraintCode::Uv:
5864 case InlineAsm::ConstraintCode::Uy:
5868 OutOps.push_back(
Op);
5879 return new ARMDAGToDAGISelLegacy(TM, OptLevel);
unsigned const MachineRegisterInfo * MRI
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
static bool isThumb(const MCSubtargetInfo &STI)
static unsigned getVectorShuffleOpcode(EVT VT, unsigned Opc64[3], unsigned Opc128[3])
static int getBankedRegisterMask(StringRef RegString)
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
static bool isVSTfixed(unsigned Opc)
static bool isVLDfixed(unsigned Opc)
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static std::optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
static bool shouldUseZeroOffsetLdSt(SDValue N)
static int getMClassFlagsMask(StringRef Flags)
static bool SDValueToConstBool(SDValue SDVal)
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
This file describes how to lower LLVM code to machine code.
support::ulittle16_t & Lo
support::ulittle16_t & Hi
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
An arbitrary precision integer that knows its signedness.
bool isThumb1Only() const
bool hasFPARMv8Base() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
uint64_t getZExtValue() const
This class represents an Operation in the Expression.
Container class for subtarget features.
FunctionPass class - This class is used to implement most global optimizations.
This class is used to form a handle around another node that is persistent and is updated across invo...
Base class for LoadSDNode and StoreSDNode.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
unsigned getOpcode() const
Return the opcode number for this descriptor.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MLOAD node.
This is an abstract virtual class for memory operations.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
int getNodeId() const
Return the unique node id.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string lower() const
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt32Ty(LLVMContext &C)
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
float getFPImmFloat(unsigned Imm)
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOptLevel OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
CodeGenOptLevel
Code generation optimization level.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ And
Bitwise or logical AND of integers.
@ NearestTiesToEven
roundTiesToEven.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
These are IR-level optimization flags that may be propagated to SDNodes.