32#include "llvm/IR/IntrinsicsARM.h"
42#define DEBUG_TYPE "arm-isel"
43#define PASS_NAME "ARM Instruction Selection"
47 cl::desc(
"Disable isel of shifter-op"),
62 ARMDAGToDAGISel() =
delete;
78 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &dl) {
79 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
88 bool hasNoVMLxHazardUse(
SDNode *
N)
const;
89 bool isShifterOpProfitable(
const SDValue &Shift,
93 bool CheckProfitability =
true);
95 SDValue &
B,
bool CheckProfitability =
true);
99 return SelectRegShifterOperand(
N,
A,
B,
C,
false);
103 return SelectImmShifterOperand(
N,
A,
B,
false);
108 return SelectImmShifterOperand(
N,
A,
B,
false);
119 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
154 template <
unsigned Shift>
159 template <
unsigned Shift>
165 template <
unsigned Shift>
169 template <
unsigned Shift>
175 template<
int Min,
int Max>
178 inline bool is_so_imm(
unsigned Imm)
const {
182 inline bool is_so_imm_not(
unsigned Imm)
const {
186 inline bool is_t2_so_imm(
unsigned Imm)
const {
190 inline bool is_t2_so_imm_not(
unsigned Imm)
const {
195#include "ARMGenDAGISel.inc"
201 bool tryARMIndexedLoad(
SDNode *
N);
202 bool tryT1IndexedLoad(
SDNode *
N);
203 bool tryT2IndexedLoad(
SDNode *
N);
204 bool tryMVEIndexedLoad(
SDNode *
N);
215 void SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
223 void SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
230 void SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
231 unsigned NumVecs,
const uint16_t *DOpcodes,
235 template <
typename SDValueVector>
236 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
238 template <
typename SDValueVector>
239 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
242 template <
typename SDValueVector>
243 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc);
244 template <
typename SDValueVector>
245 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
EVT InactiveTy);
248 void SelectMVE_WB(
SDNode *
N,
const uint16_t *Opcodes,
bool Predicated);
252 bool HasSaturationOperand);
256 uint16_t OpcodeWithNoCarry,
bool Add,
bool Predicated);
260 void SelectMVE_VSHLC(
SDNode *
N,
bool Predicated);
270 void SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
272 size_t Stride,
size_t TySize);
276 void SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
280 void SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
288 void SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
289 const uint16_t *
const *Opcodes,
bool HasWriteback);
294 bool Wrapping,
bool Predicated);
308 void SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
bool isUpdating,
309 unsigned NumVecs,
const uint16_t *DOpcodes,
310 const uint16_t *QOpcodes0 =
nullptr,
311 const uint16_t *QOpcodes1 =
nullptr);
316 bool tryInsertVectorElt(
SDNode *
N);
321 bool tryReadRegister(
SDNode *
N);
322 bool tryWriteRegister(
SDNode *
N);
326 void SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI);
328 void SelectCMP_SWAP(
SDNode *
N);
334 std::vector<SDValue> &OutOps)
override;
356 bool canExtractShiftFromMul(
const SDValue &
N,
unsigned MaxShift,
357 unsigned &PowerOfTwo,
SDValue &NewMulConst)
const;
369 ID,
std::make_unique<ARMDAGToDAGISel>(tm, OptLevel)) {}
373char ARMDAGToDAGISelLegacy::ID = 0;
380 if (
N->getOpcode() ==
ISD::Constant &&
N->getValueType(0) == MVT::i32) {
381 Imm =
N->getAsZExtVal();
397 return N->getOpcode() == Opc &&
406 int RangeMin,
int RangeMax,
407 int &ScaledConstant) {
408 assert(Scale > 0 &&
"Invalid scale!");
415 ScaledConstant = (int)
C->getZExtValue();
416 if ((ScaledConstant % Scale) != 0)
419 ScaledConstant /= Scale;
420 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
423void ARMDAGToDAGISel::PreprocessISelDAG() {
424 if (!Subtarget->hasV6T2Ops())
427 bool isThumb2 = Subtarget->isThumb();
444 unsigned And_imm = 0;
454 if (TZ != 1 && TZ != 2)
466 if (And_imm & (And_imm + 1))
471 unsigned Srl_imm = 0;
482 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
485 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
486 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
493 CurDAG->getConstant(Srl_imm + TZ,
SDLoc(Srl),
497 CurDAG->getConstant(And_imm,
SDLoc(Srl), MVT::i32));
499 N1, CurDAG->getConstant(TZ,
SDLoc(Srl), MVT::i32));
500 CurDAG->UpdateNodeOperands(&
N, N0, N1);
507bool ARMDAGToDAGISel::hasNoVMLxHazardUse(
SDNode *
N)
const {
508 if (OptLevel == CodeGenOptLevel::None)
511 if (!Subtarget->hasVMLxHazards())
520 if (
Use->isMachineOpcode()) {
522 CurDAG->getSubtarget().getInstrInfo());
528 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
543 return TII->isFpMLxInstruction(Opcode);
549bool ARMDAGToDAGISel::isShifterOpProfitable(
const SDValue &Shift,
558 (ShAmt == 2 || (Subtarget->
isSwift() && ShAmt == 1));
561bool ARMDAGToDAGISel::canExtractShiftFromMul(
const SDValue &
N,
563 unsigned &PowerOfTwo,
570 if (!
N.hasOneUse())
return false;
573 if (!MulConst)
return false;
576 if (!MulConst->
hasOneUse())
return false;
578 if (MulConstVal == 0)
return false;
581 PowerOfTwo = MaxShift;
582 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
584 if (PowerOfTwo == 0)
return false;
588 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
589 NewMulConst = CurDAG->getConstant(NewMulConstVal,
SDLoc(
N), MVT::i32);
592 return NewCost < OldCost;
596 CurDAG->RepositionNode(
N.getNode()->getIterator(),
M.getNode());
600bool ARMDAGToDAGISel::SelectImmShifterOperand(
SDValue N,
603 bool CheckProfitability) {
610 unsigned PowerOfTwo = 0;
612 if (canExtractShiftFromMul(
N, 31, PowerOfTwo, NewMulConst)) {
615 replaceDAGValue(
N.getOperand(1), NewMulConst);
617 Opc = CurDAG->getTargetConstant(
629 BaseReg =
N.getOperand(0);
630 unsigned ShImmVal = 0;
632 if (!RHS)
return false;
633 ShImmVal =
RHS->getZExtValue() & 31;
639bool ARMDAGToDAGISel::SelectRegShifterOperand(
SDValue N,
643 bool CheckProfitability) {
653 BaseReg =
N.getOperand(0);
654 unsigned ShImmVal = 0;
656 if (RHS)
return false;
658 ShReg =
N.getOperand(1);
659 if (CheckProfitability && !isShifterOpProfitable(
N, ShOpcVal, ShImmVal))
671 return CurDAG->haveNoCommonBitsSet(
N, Parent->
getOperand(1));
675bool ARMDAGToDAGISel::SelectAddrModeImm12(
SDValue N,
682 !CurDAG->isBaseWithConstantOffset(
N)) {
685 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
686 Base = CurDAG->getTargetFrameIndex(
687 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
688 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
696 Base =
N.getOperand(0);
699 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
704 int RHSC = (int)
RHS->getSExtValue();
708 if (RHSC > -0x1000 && RHSC < 0x1000) {
709 Base =
N.getOperand(0);
711 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
712 Base = CurDAG->getTargetFrameIndex(
713 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
715 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
722 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
734 int RHSC = (int)
RHS->getZExtValue();
743 unsigned ShAmt =
Log2_32(RHSC);
756 !CurDAG->isBaseWithConstantOffset(
N))
763 -0x1000+1, 0x1000, RHSC))
773 Base =
N.getOperand(0);
780 dyn_cast<ConstantSDNode>(
N.getOperand(1).getOperand(1))) {
781 ShAmt = Sh->getZExtValue();
782 if (isShifterOpProfitable(
Offset, ShOpcVal, ShAmt))
783 Offset =
N.getOperand(1).getOperand(0);
796 N.getOperand(0).hasOneUse())) {
802 dyn_cast<ConstantSDNode>(
N.getOperand(0).getOperand(1))) {
803 ShAmt = Sh->getZExtValue();
804 if (isShifterOpProfitable(
N.getOperand(0), ShOpcVal, ShAmt)) {
805 Offset =
N.getOperand(0).getOperand(0);
806 Base =
N.getOperand(1);
820 unsigned PowerOfTwo = 0;
822 if (canExtractShiftFromMul(
Offset, 31, PowerOfTwo, NewMulConst)) {
824 replaceDAGValue(
Offset.getOperand(1), NewMulConst);
825 Offset = Handle.getValue();
838 unsigned Opcode =
Op->getOpcode();
840 ? cast<LoadSDNode>(
Op)->getAddressingMode()
841 : cast<StoreSDNode>(
Op)->getAddressingMode();
855 ShAmt = Sh->getZExtValue();
856 if (isShifterOpProfitable(
N, ShOpcVal, ShAmt))
874 unsigned Opcode =
Op->getOpcode();
876 ? cast<LoadSDNode>(
Op)->getAddressingMode()
877 : cast<StoreSDNode>(
Op)->getAddressingMode();
883 Offset = CurDAG->getRegister(0, MVT::i32);
884 Opc = CurDAG->getTargetConstant(Val,
SDLoc(
Op), MVT::i32);
894 unsigned Opcode =
Op->getOpcode();
896 ? cast<LoadSDNode>(
Op)->getAddressingMode()
897 : cast<StoreSDNode>(
Op)->getAddressingMode();
902 Offset = CurDAG->getRegister(0, MVT::i32);
917bool ARMDAGToDAGISel::SelectAddrMode3(
SDValue N,
922 Base =
N.getOperand(0);
929 if (!CurDAG->isBaseWithConstantOffset(
N)) {
932 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
933 Base = CurDAG->getTargetFrameIndex(
934 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
936 Offset = CurDAG->getRegister(0, MVT::i32);
945 -256 + 1, 256, RHSC)) {
946 Base =
N.getOperand(0);
948 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
949 Base = CurDAG->getTargetFrameIndex(
950 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
952 Offset = CurDAG->getRegister(0, MVT::i32);
964 Base =
N.getOperand(0);
973 unsigned Opcode =
Op->getOpcode();
975 ? cast<LoadSDNode>(
Op)->getAddressingMode()
976 : cast<StoreSDNode>(
Op)->getAddressingMode();
981 Offset = CurDAG->getRegister(0, MVT::i32);
995 if (!CurDAG->isBaseWithConstantOffset(
N)) {
998 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
999 Base = CurDAG->getTargetFrameIndex(
1000 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1005 Base =
N.getOperand(0);
1014 const int Scale =
FP16 ? 2 : 4;
1017 Base =
N.getOperand(0);
1019 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1020 Base = CurDAG->getTargetFrameIndex(
1021 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1052bool ARMDAGToDAGISel::SelectAddrMode5(
SDValue N,
1057bool ARMDAGToDAGISel::SelectAddrMode5FP16(
SDValue N,
1066 unsigned Alignment = 0;
1068 MemSDNode *MemN = cast<MemSDNode>(Parent);
1070 if (isa<LSBaseSDNode>(MemN) ||
1078 if (MMOAlign.
value() >= MemSize && MemSize > 1)
1079 Alignment = MemSize;
1087 Align = CurDAG->getTargetConstant(Alignment,
SDLoc(
N), MVT::i32);
1100 Offset = CurDAG->getRegister(0, MVT::i32);
1105bool ARMDAGToDAGISel::SelectAddrModePC(
SDValue N,
1130 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
1131 return C->getSExtValue() < 0 &&
C->getSExtValue() >= -255;
1138 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N)) {
1146 Base =
N.getOperand(0);
1155 return SelectThumbAddrModeRRSext(
N,
Base,
Offset);
1159ARMDAGToDAGISel::SelectThumbAddrModeImm5S(
SDValue N,
unsigned Scale,
1163 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1167 if (!CurDAG->isBaseWithConstantOffset(
N)) {
1175 Base =
N.getOperand(0);
1180 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1187 Base =
N.getOperand(0);
1188 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1199 return SelectThumbAddrModeImm5S(
N, 4,
Base, OffImm);
1205 return SelectThumbAddrModeImm5S(
N, 2,
Base, OffImm);
1211 return SelectThumbAddrModeImm5S(
N, 1,
Base, OffImm);
1214bool ARMDAGToDAGISel::SelectThumbAddrModeSP(
SDValue N,
1217 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1223 Base = CurDAG->getTargetFrameIndex(
1224 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1225 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1229 if (!CurDAG->isBaseWithConstantOffset(
N))
1236 Base =
N.getOperand(0);
1237 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1248 Base = CurDAG->getTargetFrameIndex(
1249 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1250 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1260template <
unsigned Shift>
1263 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1267 Base =
N.getOperand(0);
1271 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1278 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1288bool ARMDAGToDAGISel::SelectT2AddrModeImm12(
SDValue N,
1294 !CurDAG->isBaseWithConstantOffset(
N)) {
1297 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1298 Base = CurDAG->getTargetFrameIndex(
1299 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1300 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1308 Base =
N.getOperand(0);
1313 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1317 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1318 if (SelectT2AddrModeImm8(
N,
Base, OffImm))
1322 int RHSC = (int)
RHS->getZExtValue();
1326 if (RHSC >= 0 && RHSC < 0x1000) {
1327 Base =
N.getOperand(0);
1329 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1330 Base = CurDAG->getTargetFrameIndex(
1331 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1333 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1340 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1344template <
unsigned Shift>
1347 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1350 Base =
N.getOperand(0);
1352 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1353 Base = CurDAG->getTargetFrameIndex(
1354 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1360 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1367 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1371bool ARMDAGToDAGISel::SelectT2AddrModeImm8(
SDValue N,
1375 !CurDAG->isBaseWithConstantOffset(
N))
1378 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1379 int RHSC = (int)
RHS->getSExtValue();
1383 if ((RHSC >= -255) && (RHSC < 0)) {
1384 Base =
N.getOperand(0);
1386 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1387 Base = CurDAG->getTargetFrameIndex(
1388 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1390 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1400 unsigned Opcode =
Op->getOpcode();
1402 ? cast<LoadSDNode>(
Op)->getAddressingMode()
1403 : cast<StoreSDNode>(
Op)->getAddressingMode();
1407 ? CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32)
1408 : CurDAG->getTargetConstant(-RHSC,
SDLoc(
N), MVT::i32);
1415template <
unsigned Shift>
1418 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1422 Base =
N.getOperand(0);
1424 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1425 Base = CurDAG->getTargetFrameIndex(
1426 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1432 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1439 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1443template <
unsigned Shift>
1446 return SelectT2AddrModeImm7Offset(
Op,
N, OffImm, Shift);
1452 unsigned Opcode =
Op->getOpcode();
1456 AM = cast<LoadSDNode>(
Op)->getAddressingMode();
1459 AM = cast<StoreSDNode>(
Op)->getAddressingMode();
1462 AM = cast<MaskedLoadSDNode>(
Op)->getAddressingMode();
1465 AM = cast<MaskedStoreSDNode>(
Op)->getAddressingMode();
1476 ? CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32)
1477 : CurDAG->getTargetConstant(-RHSC * (1 << Shift),
SDLoc(
N),
1484template <
int Min,
int Max>
1485bool ARMDAGToDAGISel::SelectImmediateInRange(
SDValue N,
SDValue &OffImm) {
1488 OffImm = CurDAG->getTargetConstant(Val,
SDLoc(
N), MVT::i32);
1494bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(
SDValue N,
1498 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N))
1502 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1503 int RHSC = (int)
RHS->getZExtValue();
1504 if (RHSC >= 0 && RHSC < 0x1000)
1506 else if (RHSC < 0 && RHSC >= -255)
1512 Base =
N.getOperand(0);
1513 OffReg =
N.getOperand(1);
1527 ShAmt = Sh->getZExtValue();
1528 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1539 unsigned PowerOfTwo = 0;
1541 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1543 replaceDAGValue(OffReg.
getOperand(1), NewMulConst);
1549 ShImm = CurDAG->getTargetConstant(ShAmt,
SDLoc(
N), MVT::i32);
1559 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1561 if (
N.getOpcode() !=
ISD::ADD || !CurDAG->isBaseWithConstantOffset(
N))
1569 if (RHSC > 1020 || RHSC % 4 != 0)
1572 Base =
N.getOperand(0);
1574 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1575 Base = CurDAG->getTargetFrameIndex(
1576 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1579 OffImm = CurDAG->getTargetConstant(RHSC/4,
SDLoc(
N), MVT::i32);
1590void ARMDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
1592 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {
MemOp});
1595bool ARMDAGToDAGISel::tryARMIndexedLoad(
SDNode *
N) {
1601 EVT LoadedVT =
LD->getMemoryVT();
1604 unsigned Opcode = 0;
1606 if (LoadedVT == MVT::i32 && isPre &&
1607 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1608 Opcode = ARM::LDR_PRE_IMM;
1610 }
else if (LoadedVT == MVT::i32 && !isPre &&
1611 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1612 Opcode = ARM::LDR_POST_IMM;
1614 }
else if (LoadedVT == MVT::i32 &&
1615 SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1616 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1619 }
else if (LoadedVT == MVT::i16 &&
1620 SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1623 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1624 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1625 }
else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1627 if (SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1629 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1633 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1635 Opcode = ARM::LDRB_PRE_IMM;
1636 }
else if (!isPre &&
1637 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1639 Opcode = ARM::LDRB_POST_IMM;
1640 }
else if (SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1642 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1648 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1652 CurDAG->getRegister(0, MVT::i32), Chain };
1653 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1655 transferMemOperands(
N, New);
1656 ReplaceNode(
N, New);
1662 CurDAG->getRegister(0, MVT::i32), Chain };
1663 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1665 transferMemOperands(
N, New);
1666 ReplaceNode(
N, New);
1674bool ARMDAGToDAGISel::tryT1IndexedLoad(
SDNode *
N) {
1676 EVT LoadedVT =
LD->getMemoryVT();
1682 auto *COffs = dyn_cast<ConstantSDNode>(
LD->getOffset());
1683 if (!COffs || COffs->getZExtValue() != 4)
1693 CurDAG->getRegister(0, MVT::i32), Chain };
1694 SDNode *
New = CurDAG->getMachineNode(ARM::tLDR_postidx,
SDLoc(
N), MVT::i32,
1695 MVT::i32, MVT::Other, Ops);
1696 transferMemOperands(
N, New);
1697 ReplaceNode(
N, New);
1701bool ARMDAGToDAGISel::tryT2IndexedLoad(
SDNode *
N) {
1707 EVT LoadedVT =
LD->getMemoryVT();
1711 unsigned Opcode = 0;
1713 if (SelectT2AddrModeImm8Offset(
N,
LD->getOffset(),
Offset)) {
1716 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1720 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1722 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1727 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1729 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1741 CurDAG->getRegister(0, MVT::i32), Chain };
1742 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1744 transferMemOperands(
N, New);
1745 ReplaceNode(
N, New);
1752bool ARMDAGToDAGISel::tryMVEIndexedLoad(
SDNode *
N) {
1754 unsigned Opcode = 0;
1755 bool isSExtLd, isPre;
1765 LoadedVT =
LD->getMemoryVT();
1769 Chain =
LD->getChain();
1772 Alignment =
LD->getAlign();
1776 PredReg = CurDAG->getRegister(0, MVT::i32);
1781 LoadedVT =
LD->getMemoryVT();
1785 Chain =
LD->getChain();
1788 Alignment =
LD->getAlign();
1792 PredReg =
LD->getMask();
1799 bool CanChangeType = Subtarget->
isLittle() && !isa<MaskedLoadSDNode>(
N);
1802 if (Alignment >=
Align(2) && LoadedVT == MVT::v4i16 &&
1803 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1)) {
1805 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1807 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1808 }
else if (LoadedVT == MVT::v8i8 &&
1809 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1811 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1813 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1814 }
else if (LoadedVT == MVT::v4i8 &&
1815 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1817 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1819 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1820 }
else if (Alignment >=
Align(4) &&
1821 (CanChangeType || LoadedVT == MVT::v4i32 ||
1822 LoadedVT == MVT::v4f32) &&
1823 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 2))
1824 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1825 else if (Alignment >=
Align(2) &&
1826 (CanChangeType || LoadedVT == MVT::v8i16 ||
1827 LoadedVT == MVT::v8f16) &&
1828 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1))
1829 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1830 else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
1831 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0))
1832 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1838 CurDAG->getTargetConstant(Pred,
SDLoc(
N), MVT::i32),
1840 CurDAG->getRegister(0, MVT::i32),
1843 N->getValueType(0), MVT::Other, Ops);
1844 transferMemOperands(
N, New);
1848 CurDAG->RemoveDeadNode(
N);
1856 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1857 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1858 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1859 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1860 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1867 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1868 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1869 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1870 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1871 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1877 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1879 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1880 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1881 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1882 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1888 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1890 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1891 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1892 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1893 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1901 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1902 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1903 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1904 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1905 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1906 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1907 V2, SubReg2, V3, SubReg3 };
1908 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1915 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1917 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1918 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1919 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1920 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1921 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1922 V2, SubReg2, V3, SubReg3 };
1923 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1930 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1932 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1933 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1934 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1935 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1936 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1937 V2, SubReg2, V3, SubReg3 };
1938 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1945 unsigned NumVecs,
bool is64BitVector) {
1946 unsigned NumRegs = NumVecs;
1947 if (!is64BitVector && NumVecs < 3)
1950 unsigned Alignment =
Align->getAsZExtVal();
1951 if (Alignment >= 32 && NumRegs == 4)
1953 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1955 else if (Alignment >= 8)
1960 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1966 default:
return false;
1967 case ARM::VLD1d8wb_fixed :
return true;
1968 case ARM::VLD1d16wb_fixed :
return true;
1969 case ARM::VLD1d64Qwb_fixed :
return true;
1970 case ARM::VLD1d32wb_fixed :
return true;
1971 case ARM::VLD1d64wb_fixed :
return true;
1972 case ARM::VLD1d8TPseudoWB_fixed :
return true;
1973 case ARM::VLD1d16TPseudoWB_fixed :
return true;
1974 case ARM::VLD1d32TPseudoWB_fixed :
return true;
1975 case ARM::VLD1d64TPseudoWB_fixed :
return true;
1976 case ARM::VLD1d8QPseudoWB_fixed :
return true;
1977 case ARM::VLD1d16QPseudoWB_fixed :
return true;
1978 case ARM::VLD1d32QPseudoWB_fixed :
return true;
1979 case ARM::VLD1d64QPseudoWB_fixed :
return true;
1980 case ARM::VLD1q8wb_fixed :
return true;
1981 case ARM::VLD1q16wb_fixed :
return true;
1982 case ARM::VLD1q32wb_fixed :
return true;
1983 case ARM::VLD1q64wb_fixed :
return true;
1984 case ARM::VLD1DUPd8wb_fixed :
return true;
1985 case ARM::VLD1DUPd16wb_fixed :
return true;
1986 case ARM::VLD1DUPd32wb_fixed :
return true;
1987 case ARM::VLD1DUPq8wb_fixed :
return true;
1988 case ARM::VLD1DUPq16wb_fixed :
return true;
1989 case ARM::VLD1DUPq32wb_fixed :
return true;
1990 case ARM::VLD2d8wb_fixed :
return true;
1991 case ARM::VLD2d16wb_fixed :
return true;
1992 case ARM::VLD2d32wb_fixed :
return true;
1993 case ARM::VLD2q8PseudoWB_fixed :
return true;
1994 case ARM::VLD2q16PseudoWB_fixed :
return true;
1995 case ARM::VLD2q32PseudoWB_fixed :
return true;
1996 case ARM::VLD2DUPd8wb_fixed :
return true;
1997 case ARM::VLD2DUPd16wb_fixed :
return true;
1998 case ARM::VLD2DUPd32wb_fixed :
return true;
1999 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return true;
2000 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return true;
2001 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return true;
2008 default:
return false;
2009 case ARM::VST1d8wb_fixed :
return true;
2010 case ARM::VST1d16wb_fixed :
return true;
2011 case ARM::VST1d32wb_fixed :
return true;
2012 case ARM::VST1d64wb_fixed :
return true;
2013 case ARM::VST1q8wb_fixed :
return true;
2014 case ARM::VST1q16wb_fixed :
return true;
2015 case ARM::VST1q32wb_fixed :
return true;
2016 case ARM::VST1q64wb_fixed :
return true;
2017 case ARM::VST1d8TPseudoWB_fixed :
return true;
2018 case ARM::VST1d16TPseudoWB_fixed :
return true;
2019 case ARM::VST1d32TPseudoWB_fixed :
return true;
2020 case ARM::VST1d64TPseudoWB_fixed :
return true;
2021 case ARM::VST1d8QPseudoWB_fixed :
return true;
2022 case ARM::VST1d16QPseudoWB_fixed :
return true;
2023 case ARM::VST1d32QPseudoWB_fixed :
return true;
2024 case ARM::VST1d64QPseudoWB_fixed :
return true;
2025 case ARM::VST2d8wb_fixed :
return true;
2026 case ARM::VST2d16wb_fixed :
return true;
2027 case ARM::VST2d32wb_fixed :
return true;
2028 case ARM::VST2q8PseudoWB_fixed :
return true;
2029 case ARM::VST2q16PseudoWB_fixed :
return true;
2030 case ARM::VST2q32PseudoWB_fixed :
return true;
2038 &&
"Incorrect fixed stride updating instruction.");
2041 case ARM::VLD1d8wb_fixed:
return ARM::VLD1d8wb_register;
2042 case ARM::VLD1d16wb_fixed:
return ARM::VLD1d16wb_register;
2043 case ARM::VLD1d32wb_fixed:
return ARM::VLD1d32wb_register;
2044 case ARM::VLD1d64wb_fixed:
return ARM::VLD1d64wb_register;
2045 case ARM::VLD1q8wb_fixed:
return ARM::VLD1q8wb_register;
2046 case ARM::VLD1q16wb_fixed:
return ARM::VLD1q16wb_register;
2047 case ARM::VLD1q32wb_fixed:
return ARM::VLD1q32wb_register;
2048 case ARM::VLD1q64wb_fixed:
return ARM::VLD1q64wb_register;
2049 case ARM::VLD1d64Twb_fixed:
return ARM::VLD1d64Twb_register;
2050 case ARM::VLD1d64Qwb_fixed:
return ARM::VLD1d64Qwb_register;
2051 case ARM::VLD1d8TPseudoWB_fixed:
return ARM::VLD1d8TPseudoWB_register;
2052 case ARM::VLD1d16TPseudoWB_fixed:
return ARM::VLD1d16TPseudoWB_register;
2053 case ARM::VLD1d32TPseudoWB_fixed:
return ARM::VLD1d32TPseudoWB_register;
2054 case ARM::VLD1d64TPseudoWB_fixed:
return ARM::VLD1d64TPseudoWB_register;
2055 case ARM::VLD1d8QPseudoWB_fixed:
return ARM::VLD1d8QPseudoWB_register;
2056 case ARM::VLD1d16QPseudoWB_fixed:
return ARM::VLD1d16QPseudoWB_register;
2057 case ARM::VLD1d32QPseudoWB_fixed:
return ARM::VLD1d32QPseudoWB_register;
2058 case ARM::VLD1d64QPseudoWB_fixed:
return ARM::VLD1d64QPseudoWB_register;
2059 case ARM::VLD1DUPd8wb_fixed :
return ARM::VLD1DUPd8wb_register;
2060 case ARM::VLD1DUPd16wb_fixed :
return ARM::VLD1DUPd16wb_register;
2061 case ARM::VLD1DUPd32wb_fixed :
return ARM::VLD1DUPd32wb_register;
2062 case ARM::VLD1DUPq8wb_fixed :
return ARM::VLD1DUPq8wb_register;
2063 case ARM::VLD1DUPq16wb_fixed :
return ARM::VLD1DUPq16wb_register;
2064 case ARM::VLD1DUPq32wb_fixed :
return ARM::VLD1DUPq32wb_register;
2065 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return ARM::VLD2DUPq8OddPseudoWB_register;
2066 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return ARM::VLD2DUPq16OddPseudoWB_register;
2067 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return ARM::VLD2DUPq32OddPseudoWB_register;
2069 case ARM::VST1d8wb_fixed:
return ARM::VST1d8wb_register;
2070 case ARM::VST1d16wb_fixed:
return ARM::VST1d16wb_register;
2071 case ARM::VST1d32wb_fixed:
return ARM::VST1d32wb_register;
2072 case ARM::VST1d64wb_fixed:
return ARM::VST1d64wb_register;
2073 case ARM::VST1q8wb_fixed:
return ARM::VST1q8wb_register;
2074 case ARM::VST1q16wb_fixed:
return ARM::VST1q16wb_register;
2075 case ARM::VST1q32wb_fixed:
return ARM::VST1q32wb_register;
2076 case ARM::VST1q64wb_fixed:
return ARM::VST1q64wb_register;
2077 case ARM::VST1d8TPseudoWB_fixed:
return ARM::VST1d8TPseudoWB_register;
2078 case ARM::VST1d16TPseudoWB_fixed:
return ARM::VST1d16TPseudoWB_register;
2079 case ARM::VST1d32TPseudoWB_fixed:
return ARM::VST1d32TPseudoWB_register;
2080 case ARM::VST1d64TPseudoWB_fixed:
return ARM::VST1d64TPseudoWB_register;
2081 case ARM::VST1d8QPseudoWB_fixed:
return ARM::VST1d8QPseudoWB_register;
2082 case ARM::VST1d16QPseudoWB_fixed:
return ARM::VST1d16QPseudoWB_register;
2083 case ARM::VST1d32QPseudoWB_fixed:
return ARM::VST1d32QPseudoWB_register;
2084 case ARM::VST1d64QPseudoWB_fixed:
return ARM::VST1d64QPseudoWB_register;
2086 case ARM::VLD2d8wb_fixed:
return ARM::VLD2d8wb_register;
2087 case ARM::VLD2d16wb_fixed:
return ARM::VLD2d16wb_register;
2088 case ARM::VLD2d32wb_fixed:
return ARM::VLD2d32wb_register;
2089 case ARM::VLD2q8PseudoWB_fixed:
return ARM::VLD2q8PseudoWB_register;
2090 case ARM::VLD2q16PseudoWB_fixed:
return ARM::VLD2q16PseudoWB_register;
2091 case ARM::VLD2q32PseudoWB_fixed:
return ARM::VLD2q32PseudoWB_register;
2093 case ARM::VST2d8wb_fixed:
return ARM::VST2d8wb_register;
2094 case ARM::VST2d16wb_fixed:
return ARM::VST2d16wb_register;
2095 case ARM::VST2d32wb_fixed:
return ARM::VST2d32wb_register;
2096 case ARM::VST2q8PseudoWB_fixed:
return ARM::VST2q8PseudoWB_register;
2097 case ARM::VST2q16PseudoWB_fixed:
return ARM::VST2q16PseudoWB_register;
2098 case ARM::VST2q32PseudoWB_fixed:
return ARM::VST2q32PseudoWB_register;
2100 case ARM::VLD2DUPd8wb_fixed:
return ARM::VLD2DUPd8wb_register;
2101 case ARM::VLD2DUPd16wb_fixed:
return ARM::VLD2DUPd16wb_register;
2102 case ARM::VLD2DUPd32wb_fixed:
return ARM::VLD2DUPd32wb_register;
2111 auto C = dyn_cast<ConstantSDNode>(Inc);
2115void ARMDAGToDAGISel::SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2119 assert(Subtarget->hasNEON());
2120 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLD NumVecs out-of-range");
2124 bool IsIntrinsic = !isUpdating;
2126 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2127 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2131 EVT VT =
N->getValueType(0);
2133 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2161 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2166 std::vector<EVT> ResTys;
2167 ResTys.push_back(ResTy);
2169 ResTys.push_back(MVT::i32);
2170 ResTys.push_back(MVT::Other);
2173 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2178 if (is64BitVector || NumVecs <= 2) {
2179 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2184 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2200 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2210 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2211 const SDValue OpsA[] = { MemAddr,
Align, Reg0, ImplDef, Pred, Reg0, Chain };
2213 ResTy, AddrTy, MVT::Other, OpsA);
2220 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2222 "only constant post-increment update allowed for VLD3/4");
2230 VLd = CurDAG->getMachineNode(QOpcodes1[
OpcodeIndex], dl, ResTys, Ops);
2235 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {
MemOp});
2238 ReplaceNode(
N, VLd);
2244 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2245 ARM::qsub_3 == ARM::qsub_0 + 3,
2246 "Unexpected subreg numbering");
2247 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2248 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2250 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2254 CurDAG->RemoveDeadNode(
N);
2257void ARMDAGToDAGISel::SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2261 assert(Subtarget->hasNEON());
2262 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VST NumVecs out-of-range");
2266 bool IsIntrinsic = !isUpdating;
2268 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2269 unsigned Vec0Idx = 3;
2270 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2276 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2278 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2302 std::vector<EVT> ResTys;
2304 ResTys.push_back(MVT::i32);
2305 ResTys.push_back(MVT::Other);
2308 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2312 if (is64BitVector || NumVecs <= 2) {
2315 SrcReg =
N->getOperand(Vec0Idx);
2316 }
else if (is64BitVector) {
2318 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2319 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2321 SrcReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2327 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2328 :
N->getOperand(Vec0Idx + 3);
2329 SrcReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2333 SDValue Q0 =
N->getOperand(Vec0Idx);
2334 SDValue Q1 =
N->getOperand(Vec0Idx + 1);
2335 SrcReg =
SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2338 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2343 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2361 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2364 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {
MemOp});
2366 ReplaceNode(
N, VSt);
2374 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2375 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2378 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2379 :
N->getOperand(Vec0Idx + 3);
2380 SDValue RegSeq =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2384 const SDValue OpsA[] = { MemAddr,
Align, Reg0, RegSeq, Pred, Reg0, Chain };
2388 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {
MemOp});
2395 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2397 "only constant post-increment update allowed for VST3/4");
2407 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {
MemOp});
2408 ReplaceNode(
N, VStB);
2411void ARMDAGToDAGISel::SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
2415 assert(Subtarget->hasNEON());
2416 assert(NumVecs >=2 && NumVecs <= 4 &&
"VLDSTLane NumVecs out-of-range");
2420 bool IsIntrinsic = !isUpdating;
2422 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2423 unsigned Vec0Idx = 3;
2424 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2430 unsigned Lane =
N->getConstantOperandVal(Vec0Idx + NumVecs);
2431 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2434 unsigned Alignment = 0;
2436 Alignment =
Align->getAsZExtVal();
2438 if (Alignment > NumBytes)
2439 Alignment = NumBytes;
2440 if (Alignment < 8 && Alignment < NumBytes)
2443 Alignment = (Alignment & -Alignment);
2447 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2467 std::vector<EVT> ResTys;
2469 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2473 MVT::i64, ResTyElts));
2476 ResTys.push_back(MVT::i32);
2477 ResTys.push_back(MVT::Other);
2480 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2486 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2489 Ops.
push_back(IsImmUpdate ? Reg0 : Inc);
2493 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2494 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2497 SuperReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2499 SuperReg =
SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2503 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2504 :
N->getOperand(Vec0Idx + 3);
2506 SuperReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2508 SuperReg =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2516 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2518 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2519 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {
MemOp});
2521 ReplaceNode(
N, VLdLn);
2527 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2528 ARM::qsub_3 == ARM::qsub_0 + 3,
2529 "Unexpected subreg numbering");
2530 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2531 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2533 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2537 CurDAG->RemoveDeadNode(
N);
2540template <
typename SDValueVector>
2541void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2543 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2544 Ops.push_back(PredicateMask);
2545 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2548template <
typename SDValueVector>
2549void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2552 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2553 Ops.push_back(PredicateMask);
2554 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2555 Ops.push_back(Inactive);
2558template <
typename SDValueVector>
2559void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc) {
2560 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2561 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2562 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2565template <
typename SDValueVector>
2566void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2568 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2569 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2570 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2572 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2581 switch (
N->getValueType(1).getVectorElementType().getSizeInBits()) {
2583 Opcode = Opcodes[0];
2586 Opcode = Opcodes[1];
2594 int32_t ImmValue =
N->getConstantOperandVal(3);
2595 Ops.
push_back(getI32Imm(ImmValue, Loc));
2598 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2600 AddEmptyMVEPredicateToOps(Ops, Loc);
2613 transferMemOperands(
N, New);
2614 CurDAG->RemoveDeadNode(
N);
2619 bool HasSaturationOperand) {
2629 int32_t ImmValue =
N->getConstantOperandVal(3);
2630 Ops.
push_back(getI32Imm(ImmValue, Loc));
2636 if (HasSaturationOperand) {
2637 int32_t SatOp =
N->getConstantOperandVal(4);
2638 int SatBit = (SatOp == 64 ? 0 : 1);
2645 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
2647 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2650void ARMDAGToDAGISel::SelectMVE_VADCSBC(
SDNode *
N,
uint16_t OpcodeWithCarry,
2652 bool Add,
bool Predicated) {
2657 unsigned FirstInputOp = Predicated ? 2 : 1;
2661 Ops.
push_back(
N->getOperand(FirstInputOp + 1));
2662 SDValue CarryIn =
N->getOperand(FirstInputOp + 2);
2663 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2666 if (CarryInConstant &&
2667 (CarryInConstant->
getZExtValue() & CarryMask) == CarryExpected) {
2668 Opcode = OpcodeWithNoCarry;
2671 Opcode = OpcodeWithCarry;
2675 AddMVEPredicateToOps(Ops, Loc,
2676 N->getOperand(FirstInputOp + 3),
2677 N->getOperand(FirstInputOp - 1));
2679 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2681 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2684void ARMDAGToDAGISel::SelectMVE_VSHLC(
SDNode *
N,
bool Predicated) {
2692 int32_t ImmValue =
N->getConstantOperandVal(3);
2693 Ops.
push_back(getI32Imm(ImmValue, Loc));
2696 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2698 AddEmptyMVEPredicateToOps(Ops, Loc);
2700 CurDAG->SelectNodeTo(
N, ARM::MVE_VSHLC,
N->getVTList(),
ArrayRef(Ops));
2704 assert(isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant");
2711void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2714 size_t Stride,
size_t TySize) {
2715 assert(TySize < Stride &&
"Invalid TySize");
2721 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2723 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2726 auto OpIsZero = [
N](
size_t OpNo) {
2732 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2734 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2736 Opcodes += 4 * Stride;
2738 Opcodes += 2 * Stride;
2755 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(8));
2757 AddEmptyMVEPredicateToOps(Ops, Loc);
2759 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2762void ARMDAGToDAGISel::SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2765 EVT VecTy =
N->getOperand(6).getValueType();
2778 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2781void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
2785 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2787 "bad vector element size");
2788 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 1, 0);
2791void ARMDAGToDAGISel::SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
2793 bool HasWriteback) {
2794 EVT VT =
N->getValueType(0);
2800 OurOpcodes = Opcodes[0];
2803 OurOpcodes = Opcodes[1];
2806 OurOpcodes = Opcodes[2];
2814 unsigned PtrOperand = HasWriteback ? 1 : 2;
2817 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2820 for (
unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2821 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2823 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2830 ResultTys = {DataTy, MVT::i32, MVT::Other};
2831 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2833 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2837 for (i = 0; i < NumVecs; i++)
2839 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2844 CurDAG->RemoveDeadNode(
N);
2847void ARMDAGToDAGISel::SelectMVE_VxDUP(
SDNode *
N,
const uint16_t *Opcodes,
2848 bool Wrapping,
bool Predicated) {
2849 EVT VT =
N->getValueType(0);
2855 Opcode = Opcodes[0];
2858 Opcode = Opcodes[1];
2861 Opcode = Opcodes[2];
2872 Inactive =
N->getOperand(OpIdx++);
2878 SDValue ImmOp =
N->getOperand(OpIdx++);
2880 Ops.
push_back(getI32Imm(ImmValue, Loc));
2883 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(OpIdx), Inactive);
2885 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2887 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2891 size_t NumExtraOps,
bool HasAccum) {
2892 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2899 SDValue ImmCorpoc =
N->getOperand(OpIdx++);
2901 Ops.
push_back(getI32Imm(ImmCoprocVal, Loc));
2906 SDValue AccLo =
N->getOperand(OpIdx++);
2907 SDValue AccHi =
N->getOperand(OpIdx++);
2914 for (
size_t I = 0;
I < NumExtraOps;
I++)
2925 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2931 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops);
2938 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2942 for (
size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2943 if (
SDValue(
N, ResIdx).use_empty())
2945 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2946 MVT::i32, ResultPair);
2950 CurDAG->RemoveDeadNode(
N);
2953void ARMDAGToDAGISel::SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
2954 bool isUpdating,
unsigned NumVecs,
2958 assert(Subtarget->hasNEON());
2959 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLDDup NumVecs out-of-range");
2963 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2964 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2968 EVT VT =
N->getValueType(0);
2971 unsigned Alignment = 0;
2973 Alignment =
Align->getAsZExtVal();
2975 if (Alignment > NumBytes)
2976 Alignment = NumBytes;
2977 if (Alignment < 8 && Alignment < NumBytes)
2980 Alignment = (Alignment & -Alignment);
2984 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
3006 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
3011 std::vector<EVT> ResTys;
3012 ResTys.push_back(ResTy);
3014 ResTys.push_back(MVT::i32);
3015 ResTys.push_back(MVT::Other);
3018 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3023 unsigned Opc = is64BitVector ? DOpcodes[
OpcodeIndex]
3039 if (is64BitVector || NumVecs == 1) {
3043 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3044 const SDValue OpsA[] = {MemAddr,
Align, ImplDef, Pred, Reg0, Chain};
3055 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3059 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {
MemOp});
3066 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7,
"Unexpected subreg numbering");
3067 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3068 for (
unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3070 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3076 CurDAG->RemoveDeadNode(
N);
3079bool ARMDAGToDAGISel::tryInsertVectorElt(
SDNode *
N) {
3080 if (!Subtarget->hasMVEIntegerOps())
3094 (VT != MVT::v8f16 && VT != MVT::v8i16) || (Ins2.
getValueType() != VT))
3099 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3126 ExtractLane1 == ExtractLane2 + 1) {
3127 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3128 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val1.
getOperand(0));
3129 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3130 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.
getOperand(0),
3132 ReplaceUses(Ins1, NewIns);
3138 if (VT == MVT::v8i16 && Subtarget->hasFullFP16()) {
3139 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3140 ARM::ssub_0 + ExtractLane1 / 2, dl, MVT::f32, Val1.
getOperand(0));
3141 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3142 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val2.
getOperand(0));
3143 if (ExtractLane1 % 2 != 0)
3144 Inp1 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0);
3145 if (ExtractLane2 % 2 != 0)
3146 Inp2 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0);
3147 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1);
3149 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3151 ReplaceUses(Ins1, NewIns);
3158 if (VT == MVT::v8f16 && Subtarget->hasFullFP16()) {
3159 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1);
3161 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3163 ReplaceUses(Ins1, NewIns);
3170bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(
SDNode *
N,
3173 bool FixedToFloat) {
3174 auto Type =
N->getValueType(0);
3176 if (ScalarBits > 32)
3182 if (ScalarBits == 16 && !FMulFlags.
hasNoInfs() && IsUnsigned)
3207 if (!isa<ConstantSDNode>(ImmNode.
getOperand(0)))
3213 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3214 APInt(ScalarBits, Imm));
3231 if (!ImmAPF.getExactInverse(&ToConvert))
3234 APSInt Converted(64,
false);
3238 if (!IsExact || !Converted.isPowerOf2())
3241 unsigned FracBits = Converted.logBase2();
3242 if (FracBits > ScalarBits)
3246 VecVal, CurDAG->getConstant(FracBits,
SDLoc(
N), MVT::i32)};
3247 AddEmptyMVEPredicateToOps(Ops,
SDLoc(
N),
Type);
3249 unsigned int Opcode;
3250 switch (ScalarBits) {
3253 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3255 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3259 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3261 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3268 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
SDLoc(
N),
Type, Ops));
3272bool ARMDAGToDAGISel::tryFP_TO_INT(
SDNode *
N,
SDLoc dl) {
3274 if (!Subtarget->hasMVEFloatOps())
3277 if (!
Type.isVector())
3288 if (
Node->getOperand(0) !=
Node->getOperand(1))
3293 if (ScalarBits == 16 && !
Flags.hasNoInfs() && IsUnsigned)
3297 switch (ScalarBits) {
3299 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3302 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3306 CurDAG->getConstant(1, dl, MVT::i32)};
3307 AddEmptyMVEPredicateToOps(Ops, dl,
Type);
3309 ReplaceNode(
N, CurDAG->getMachineNode(Opcode, dl,
Type, Ops));
3316 return transformFixedFloatingPointConversion(
N,
Node, IsUnsigned,
false);
3319bool ARMDAGToDAGISel::tryFMULFixed(
SDNode *
N,
SDLoc dl) {
3321 if (!Subtarget->hasMVEFloatOps())
3323 auto Type =
N->getValueType(0);
3324 if (!
Type.isVector())
3327 auto LHS =
N->getOperand(0);
3331 return transformFixedFloatingPointConversion(
3335bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned) {
3336 if (!Subtarget->hasV6T2Ops())
3340 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3341 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3345 unsigned And_imm = 0;
3350 if (And_imm & (And_imm + 1))
3353 unsigned Srl_imm = 0;
3356 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3361 And_imm &= -1U >> Srl_imm;
3365 unsigned LSB = Srl_imm;
3367 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3369 if ((LSB + Width + 1) ==
N->getValueType(0).getSizeInBits()) {
3371 if (Subtarget->isThumb()) {
3372 Opc =
isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3373 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3374 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3375 getAL(CurDAG, dl), Reg0, Reg0 };
3376 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3385 SDValue Ops[] = {
N->getOperand(0).getOperand(0), ShOpc,
3386 getAL(CurDAG, dl), Reg0, Reg0 };
3387 CurDAG->SelectNodeTo(
N, ARM::MOVsi, MVT::i32, Ops);
3391 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3392 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3393 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3394 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3395 getAL(CurDAG, dl), Reg0 };
3396 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3404 unsigned Shl_imm = 0;
3406 assert(Shl_imm > 0 && Shl_imm < 32 &&
"bad amount in shift node!");
3407 unsigned Srl_imm = 0;
3409 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3411 unsigned Width = 32 - Srl_imm - 1;
3412 int LSB = Srl_imm - Shl_imm;
3415 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3416 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3417 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3418 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3419 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3420 getAL(CurDAG, dl), Reg0 };
3421 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3429 unsigned Srl_imm = 0;
3433 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3436 unsigned Width = MSB - LSB;
3437 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3438 assert(Srl_imm + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3439 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3440 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
3441 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3442 getAL(CurDAG, dl), Reg0 };
3443 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3449 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3455 if (LSB + Width > 32)
3458 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3459 assert(LSB + Width <= 32 &&
"Shouldn't create an invalid ubfx");
3460 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3461 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3462 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
3463 getAL(CurDAG, dl), Reg0 };
3464 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3481bool ARMDAGToDAGISel::tryABSOp(
SDNode *
N){
3482 SDValue SUBSrc0 =
N->getOperand(0);
3483 SDValue SUBSrc1 =
N->getOperand(1);
3484 EVT VT =
N->getValueType(0);
3500 if (XORSrc1 == SUBSrc1 && XORSrc0 == SRASrc0 && XType.
isInteger() &&
3502 unsigned Opcode = Subtarget->
isThumb2() ? ARM::t2ABS : ARM::ABS;
3503 CurDAG->SelectNodeTo(
N, Opcode, VT, XORSrc0);
3511void ARMDAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3513 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3514 if (MemTy == MVT::i8)
3515 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3516 else if (MemTy == MVT::i16)
3517 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3518 else if (MemTy == MVT::i32)
3519 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32;
3523 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3525 SDNode *CmpSwap = CurDAG->getMachineNode(
3527 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
3530 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3534 CurDAG->RemoveDeadNode(
N);
3537static std::optional<std::pair<unsigned, unsigned>>
3539 unsigned FirstOne =
A.getBitWidth() -
A.countl_zero() - 1;
3540 unsigned LastOne =
A.countr_zero();
3541 if (
A.popcount() != (FirstOne - LastOne + 1))
3542 return std::nullopt;
3543 return std::make_pair(FirstOne, LastOne);
3546void ARMDAGToDAGISel::SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI) {
3548 SwitchEQNEToPLMI =
false;
3550 if (!Subtarget->isThumb())
3557 if (!
And->hasOneUse())
3564 auto C = dyn_cast<ConstantSDNode>(
And.getOperand(1));
3578 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3579 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3580 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3581 CurDAG->getRegister(0, MVT::i32) };
3582 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3584 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
3585 CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3586 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3587 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3591 if (
Range->second == 0) {
3593 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3594 ReplaceNode(
And.getNode(), NewN);
3595 }
else if (
Range->first == 31) {
3597 NewN = EmitShift(ARM::tLSRri,
X,
Range->second);
3598 ReplaceNode(
And.getNode(), NewN);
3602 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3603 ReplaceNode(
And.getNode(), NewN);
3605 SwitchEQNEToPLMI =
true;
3606 }
else if (!Subtarget->hasV6T2Ops()) {
3609 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3610 NewN = EmitShift(ARM::tLSRri,
SDValue(NewN, 0),
3612 ReplaceNode(
And.getNode(), NewN);
3617 unsigned Opc128[3]) {
3619 "Unexpected vector shuffle length");
3632void ARMDAGToDAGISel::Select(
SDNode *
N) {
3635 if (
N->isMachineOpcode()) {
3640 switch (
N->getOpcode()) {
3662 cast<RegisterSDNode>(
Ptr.getOperand(1))->getReg() == ARM::SP &&
3663 Ptr.getOperand(0) ==
ST->getChain()) {
3665 CurDAG->getRegister(ARM::SP, MVT::i32),
3666 CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
3668 CurDAG->getRegister(0, MVT::i32),
3671 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
3673 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3674 ReplaceNode(
N, ResNode);
3681 if (tryWriteRegister(
N))
3685 if (tryReadRegister(
N))
3690 if (tryInlineAsm(
N))
3700 unsigned Val =
N->getAsZExtVal();
3703 !Subtarget->genExecuteOnly()) {
3704 SDValue CPIdx = CurDAG->getTargetConstantPool(
3706 TLI->getPointerTy(CurDAG->getDataLayout()));
3709 if (Subtarget->isThumb()) {
3713 CurDAG->getRegister(0, MVT::i32),
3714 CurDAG->getEntryNode()
3716 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
3721 CurDAG->getTargetConstant(0, dl, MVT::i32),
3723 CurDAG->getRegister(0, MVT::i32),
3724 CurDAG->getEntryNode()
3726 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
3737 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3739 ReplaceNode(
N, ResNode);
3748 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
3749 SDValue TFI = CurDAG->getTargetFrameIndex(
3750 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3757 CurDAG->SelectNodeTo(
N, ARM::tADDframe, MVT::i32, TFI,
3758 CurDAG->getTargetConstant(0, dl, MVT::i32));
3761 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3762 ARM::t2ADDri : ARM::ADDri);
3763 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3764 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3765 CurDAG->getRegister(0, MVT::i32) };
3766 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3771 if (tryInsertVectorElt(
N))
3776 if (tryV6T2BitfieldExtractOp(
N,
false))
3781 if (tryV6T2BitfieldExtractOp(
N,
true))
3788 if (tryFP_TO_INT(
N, dl))
3792 if (tryFMULFixed(
N, dl))
3799 unsigned RHSV =
C->getZExtValue();
3802 unsigned ShImm =
Log2_32(RHSV-1);
3807 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3808 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3809 if (Subtarget->isThumb()) {
3810 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3811 CurDAG->SelectNodeTo(
N, ARM::t2ADDrs, MVT::i32, Ops);
3816 CurDAG->SelectNodeTo(
N, ARM::ADDrsi, MVT::i32, Ops);
3821 unsigned ShImm =
Log2_32(RHSV+1);
3826 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3827 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3828 if (Subtarget->isThumb()) {
3829 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3830 CurDAG->SelectNodeTo(
N, ARM::t2RSBrs, MVT::i32, Ops);
3835 CurDAG->SelectNodeTo(
N, ARM::RSBrsi, MVT::i32, Ops);
3843 if (tryV6T2BitfieldExtractOp(
N,
false))
3849 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3850 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3857 bool PreferImmediateEncoding =
3858 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3859 if (!PreferImmediateEncoding &&
3865 CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
3869 CurDAG->RepositionNode(
N->getIterator(), NewImm.
getNode());
3871 if (!Subtarget->hasThumb2()) {
3872 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3873 N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3874 CurDAG->getRegister(0, MVT::i32)};
3875 ReplaceNode(
N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
3878 SDValue Ops[] = {
N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3879 CurDAG->getRegister(0, MVT::i32),
3880 CurDAG->getRegister(0, MVT::i32)};
3882 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
3893 EVT VT =
N->getValueType(0);
3896 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3898 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3901 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
3902 N1C = dyn_cast<ConstantSDNode>(N1);
3910 unsigned N1CVal = N1C->getZExtValue();
3912 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3913 (N1CVal & 0xffffU) == 0xffffU &&
3914 (N2CVal & 0xffffU) == 0x0U) {
3915 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3918 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
3919 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3927 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3928 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
3929 N->getOperand(2),
N->getOperand(3),
3931 CurDAG->getRegister(0, MVT::i32) };
3932 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
3936 if (Subtarget->isThumb()) {
3937 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3938 N->getOperand(3),
getAL(CurDAG, dl),
3939 CurDAG->getRegister(0, MVT::i32)};
3941 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
3944 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3945 N->getOperand(3),
getAL(CurDAG, dl),
3946 CurDAG->getRegister(0, MVT::i32),
3947 CurDAG->getRegister(0, MVT::i32) };
3948 ReplaceNode(
N, CurDAG->getMachineNode(
3949 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3950 MVT::i32, MVT::i32, Ops));
3955 if (Subtarget->isThumb()) {
3956 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3957 N->getOperand(3),
getAL(CurDAG, dl),
3958 CurDAG->getRegister(0, MVT::i32)};
3960 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
3963 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3964 N->getOperand(3),
getAL(CurDAG, dl),
3965 CurDAG->getRegister(0, MVT::i32),
3966 CurDAG->getRegister(0, MVT::i32) };
3967 ReplaceNode(
N, CurDAG->getMachineNode(
3968 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3969 MVT::i32, MVT::i32, Ops));
3974 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3983 if (Subtarget->isThumb())
3984 assert(Subtarget->hasThumb2() &&
3985 "This pattern should not be generated for Thumb");
3987 SDValue SmulLoHi =
N->getOperand(1);
3992 N->getOperand(1) != SmulLoHi.
getValue(1) ||
3996 unsigned Opc = Subtarget->
isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3998 N->getOperand(0),
getAL(CurDAG, dl),
3999 CurDAG->getRegister(0, MVT::i32) };
4000 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
4004 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4006 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
4007 if (tryT2IndexedLoad(
N))
4009 }
else if (Subtarget->isThumb()) {
4010 if (tryT1IndexedLoad(
N))
4012 }
else if (tryARMIndexedLoad(
N))
4018 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4023 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopSetup, dl, MVT::i32,
4025 ReplaceUses(
N, New);
4026 CurDAG->RemoveDeadNode(
N);
4030 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopStart, dl, MVT::Other,
4031 N->getOperand(1),
N->getOperand(2),
4033 ReplaceUses(
N, New);
4034 CurDAG->RemoveDeadNode(
N);
4038 SDValue Ops[] = {
N->getOperand(1),
4041 unsigned Opc = ARM::t2LoopEnd;
4042 SDNode *
New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
4043 ReplaceUses(
N, New);
4044 CurDAG->RemoveDeadNode(
N);
4051 const SDValue &Chain =
N->getOperand(0);
4053 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4054 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4060 RegOffset = CurDAG->getRegister(0, MVT::i32);
4062 SDValue Ops[] = {
Base, RegOffset, ImmOffset, Chain};
4063 SDNode *
New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4064 {MVT::Untyped, MVT::Other}, Ops);
4065 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4067 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4069 transferMemOperands(
N, New);
4073 CurDAG->RemoveDeadNode(
N);
4080 const SDValue &Chain =
N->getOperand(0);
4082 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4083 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4089 RegOffset = CurDAG->getRegister(0, MVT::i32);
4094 SDNode *
New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops);
4095 transferMemOperands(
N, New);
4097 CurDAG->RemoveDeadNode(
N);
4101 SDValue Ops[] = {
N->getOperand(1),
4105 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4106 CurDAG->getVTList(MVT::i32, MVT::Other), Ops);
4107 ReplaceUses(
N, Dec);
4108 CurDAG->RemoveDeadNode(
N);
4124 unsigned Opc = Subtarget->isThumb() ?
4125 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4143 if (
ID == Intrinsic::loop_decrement_reg) {
4145 SDValue Size = CurDAG->getTargetConstant(
Int.getConstantOperandVal(3),
4150 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4151 CurDAG->getVTList(MVT::i32, MVT::Other),
4153 ReplaceUses(
Int.getNode(), LoopDec);
4157 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
4159 ReplaceUses(
N, LoopEnd);
4160 CurDAG->RemoveDeadNode(
N);
4161 CurDAG->RemoveDeadNode(InGlue.
getNode());
4162 CurDAG->RemoveDeadNode(
Int.getNode());
4167 bool SwitchEQNEToPLMI;
4168 SelectCMPZ(InGlue.
getNode(), SwitchEQNEToPLMI);
4169 InGlue =
N->getOperand(4);
4171 if (SwitchEQNEToPLMI) {
4184 SDValue Tmp2 = CurDAG->getTargetConstant(
CC, dl, MVT::i32);
4185 SDValue Ops[] = { N1, Tmp2, N3, Chain, InGlue };
4186 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
4189 if (
N->getNumValues() == 2) {
4191 ReplaceUses(
SDValue(
N, 1), InGlue);
4195 CurDAG->RemoveDeadNode(
N);
4205 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1).getNode());
4206 if (
C &&
C->getSExtValue() < 0 && Subtarget->isThumb()) {
4207 int64_t Addend = -
C->getSExtValue();
4214 if (Addend < 1<<8) {
4216 SDValue Ops[] = {
X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4217 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
4218 CurDAG->getRegister(0, MVT::i32) };
4219 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
4221 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4222 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
X,
4223 CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4224 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
4225 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
4230 CurDAG->MorphNodeTo(
N,
ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
4241 bool SwitchEQNEToPLMI;
4242 SelectCMPZ(InGlue.
getNode(), SwitchEQNEToPLMI);
4244 if (SwitchEQNEToPLMI) {
4257 SDValue NewARMcc = CurDAG->getConstant((
unsigned)
CC, dl, MVT::i32);
4258 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), NewARMcc,
4259 N->getOperand(3),
N->getOperand(4)};
4268 EVT VT =
N->getValueType(0);
4270 unsigned Opc64[] = {ARM::VZIPd8, ARM::VZIPd16, ARM::VTRNd32};
4271 unsigned Opc128[] = {ARM::VZIPq8, ARM::VZIPq16, ARM::VZIPq32};
4274 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4275 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4276 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4280 EVT VT =
N->getValueType(0);
4282 unsigned Opc64[] = {ARM::VUZPd8, ARM::VUZPd16, ARM::VTRNd32};
4283 unsigned Opc128[] = {ARM::VUZPq8, ARM::VUZPq16, ARM::VUZPq32};
4286 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4287 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4288 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4292 EVT VT =
N->getValueType(0);
4293 unsigned Opc64[] = {ARM::VTRNd8, ARM::VTRNd16, ARM::VTRNd32};
4294 unsigned Opc128[] = {ARM::VTRNq8, ARM::VTRNq16, ARM::VTRNq32};
4297 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4298 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4299 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4303 EVT VecVT =
N->getValueType(0);
4306 if (EltVT == MVT::f64) {
4307 assert(NumElts == 2 &&
"unexpected type for BUILD_VECTOR");
4309 N, createDRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4312 assert(EltVT == MVT::f32 &&
"unexpected type for BUILD_VECTOR");
4315 N, createSRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4318 assert(NumElts == 4 &&
"unexpected type for BUILD_VECTOR");
4320 createQuadSRegsNode(VecVT,
N->getOperand(0),
N->getOperand(1),
4321 N->getOperand(2),
N->getOperand(3)));
4326 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4328 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4330 SelectVLDDup(
N,
false,
false, 1, DOpcodes, QOpcodes);
4335 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4337 SelectVLDDup(
N,
false,
false, 2, Opcodes);
4342 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4343 ARM::VLD3DUPd16Pseudo,
4344 ARM::VLD3DUPd32Pseudo };
4345 SelectVLDDup(
N,
false,
false, 3, Opcodes);
4350 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4351 ARM::VLD4DUPd16Pseudo,
4352 ARM::VLD4DUPd32Pseudo };
4353 SelectVLDDup(
N,
false,
false, 4, Opcodes);
4358 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4359 ARM::VLD1DUPd16wb_fixed,
4360 ARM::VLD1DUPd32wb_fixed };
4361 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4362 ARM::VLD1DUPq16wb_fixed,
4363 ARM::VLD1DUPq32wb_fixed };
4364 SelectVLDDup(
N,
false,
true, 1, DOpcodes, QOpcodes);
4369 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4370 ARM::VLD2DUPd16wb_fixed,
4371 ARM::VLD2DUPd32wb_fixed,
4372 ARM::VLD1q64wb_fixed };
4373 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4374 ARM::VLD2DUPq16EvenPseudo,
4375 ARM::VLD2DUPq32EvenPseudo };
4376 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4377 ARM::VLD2DUPq16OddPseudoWB_fixed,
4378 ARM::VLD2DUPq32OddPseudoWB_fixed };
4379 SelectVLDDup(
N,
false,
true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4384 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4385 ARM::VLD3DUPd16Pseudo_UPD,
4386 ARM::VLD3DUPd32Pseudo_UPD,
4387 ARM::VLD1d64TPseudoWB_fixed };
4388 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4389 ARM::VLD3DUPq16EvenPseudo,
4390 ARM::VLD3DUPq32EvenPseudo };
4391 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4392 ARM::VLD3DUPq16OddPseudo_UPD,
4393 ARM::VLD3DUPq32OddPseudo_UPD };
4394 SelectVLDDup(
N,
false,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4399 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4400 ARM::VLD4DUPd16Pseudo_UPD,
4401 ARM::VLD4DUPd32Pseudo_UPD,
4402 ARM::VLD1d64QPseudoWB_fixed };
4403 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4404 ARM::VLD4DUPq16EvenPseudo,
4405 ARM::VLD4DUPq32EvenPseudo };
4406 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4407 ARM::VLD4DUPq16OddPseudo_UPD,
4408 ARM::VLD4DUPq32OddPseudo_UPD };
4409 SelectVLDDup(
N,
false,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4414 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4415 ARM::VLD1d16wb_fixed,
4416 ARM::VLD1d32wb_fixed,
4417 ARM::VLD1d64wb_fixed };
4418 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4419 ARM::VLD1q16wb_fixed,
4420 ARM::VLD1q32wb_fixed,
4421 ARM::VLD1q64wb_fixed };
4422 SelectVLD(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4427 if (Subtarget->hasNEON()) {
4428 static const uint16_t DOpcodes[] = {
4429 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4430 ARM::VLD1q64wb_fixed};
4431 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4432 ARM::VLD2q16PseudoWB_fixed,
4433 ARM::VLD2q32PseudoWB_fixed};
4434 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4436 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4437 ARM::MVE_VLD21_8_wb};
4438 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4439 ARM::MVE_VLD21_16_wb};
4440 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4441 ARM::MVE_VLD21_32_wb};
4442 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4443 SelectMVE_VLD(
N, 2, Opcodes,
true);
4449 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4450 ARM::VLD3d16Pseudo_UPD,
4451 ARM::VLD3d32Pseudo_UPD,
4452 ARM::VLD1d64TPseudoWB_fixed};
4453 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4454 ARM::VLD3q16Pseudo_UPD,
4455 ARM::VLD3q32Pseudo_UPD };
4456 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4457 ARM::VLD3q16oddPseudo_UPD,
4458 ARM::VLD3q32oddPseudo_UPD };
4459 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4464 if (Subtarget->hasNEON()) {
4465 static const uint16_t DOpcodes[] = {
4466 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4467 ARM::VLD1d64QPseudoWB_fixed};
4468 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4469 ARM::VLD4q16Pseudo_UPD,
4470 ARM::VLD4q32Pseudo_UPD};
4471 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4472 ARM::VLD4q16oddPseudo_UPD,
4473 ARM::VLD4q32oddPseudo_UPD};
4474 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4476 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4478 ARM::MVE_VLD43_8_wb};
4479 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4481 ARM::MVE_VLD43_16_wb};
4482 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4484 ARM::MVE_VLD43_32_wb};
4485 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4486 SelectMVE_VLD(
N, 4, Opcodes,
true);
4492 if (Subtarget->hasNEON()) {
4493 static const uint16_t DOpcodes[] = {
4494 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4495 ARM::VLD1q64wb_fixed};
4496 static const uint16_t QOpcodes[] = {
4497 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4498 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4499 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4506 if (Subtarget->hasNEON()) {
4507 static const uint16_t DOpcodes[] = {
4508 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4509 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4510 static const uint16_t QOpcodes0[] = {
4511 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4512 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4513 static const uint16_t QOpcodes1[] = {
4514 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4515 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4516 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4523 if (Subtarget->hasNEON()) {
4524 static const uint16_t DOpcodes[] = {
4525 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4526 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4527 static const uint16_t QOpcodes0[] = {
4528 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4529 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4530 static const uint16_t QOpcodes1[] = {
4531 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4532 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4533 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4540 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4541 ARM::VLD2LNd16Pseudo_UPD,
4542 ARM::VLD2LNd32Pseudo_UPD };
4543 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4544 ARM::VLD2LNq32Pseudo_UPD };
4545 SelectVLDSTLane(
N,
true,
true, 2, DOpcodes, QOpcodes);
4550 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4551 ARM::VLD3LNd16Pseudo_UPD,
4552 ARM::VLD3LNd32Pseudo_UPD };
4553 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4554 ARM::VLD3LNq32Pseudo_UPD };
4555 SelectVLDSTLane(
N,
true,
true, 3, DOpcodes, QOpcodes);
4560 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4561 ARM::VLD4LNd16Pseudo_UPD,
4562 ARM::VLD4LNd32Pseudo_UPD };
4563 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4564 ARM::VLD4LNq32Pseudo_UPD };
4565 SelectVLDSTLane(
N,
true,
true, 4, DOpcodes, QOpcodes);
4570 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4571 ARM::VST1d16wb_fixed,
4572 ARM::VST1d32wb_fixed,
4573 ARM::VST1d64wb_fixed };
4574 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4575 ARM::VST1q16wb_fixed,
4576 ARM::VST1q32wb_fixed,
4577 ARM::VST1q64wb_fixed };
4578 SelectVST(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4583 if (Subtarget->hasNEON()) {
4584 static const uint16_t DOpcodes[] = {
4585 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4586 ARM::VST1q64wb_fixed};
4587 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4588 ARM::VST2q16PseudoWB_fixed,
4589 ARM::VST2q32PseudoWB_fixed};
4590 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4597 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4598 ARM::VST3d16Pseudo_UPD,
4599 ARM::VST3d32Pseudo_UPD,
4600 ARM::VST1d64TPseudoWB_fixed};
4601 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4602 ARM::VST3q16Pseudo_UPD,
4603 ARM::VST3q32Pseudo_UPD };
4604 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4605 ARM::VST3q16oddPseudo_UPD,
4606 ARM::VST3q32oddPseudo_UPD };
4607 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4612 if (Subtarget->hasNEON()) {
4613 static const uint16_t DOpcodes[] = {
4614 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4615 ARM::VST1d64QPseudoWB_fixed};
4616 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4617 ARM::VST4q16Pseudo_UPD,
4618 ARM::VST4q32Pseudo_UPD};
4619 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4620 ARM::VST4q16oddPseudo_UPD,
4621 ARM::VST4q32oddPseudo_UPD};
4622 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4629 if (Subtarget->hasNEON()) {
4630 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4631 ARM::VST1q16wb_fixed,
4632 ARM::VST1q32wb_fixed,
4633 ARM::VST1q64wb_fixed};
4634 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4635 ARM::VST1d16QPseudoWB_fixed,
4636 ARM::VST1d32QPseudoWB_fixed,
4637 ARM::VST1d64QPseudoWB_fixed };
4638 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4645 if (Subtarget->hasNEON()) {
4646 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4647 ARM::VST1d16TPseudoWB_fixed,
4648 ARM::VST1d32TPseudoWB_fixed,
4649 ARM::VST1d64TPseudoWB_fixed };
4650 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4651 ARM::VST1q16LowTPseudo_UPD,
4652 ARM::VST1q32LowTPseudo_UPD,
4653 ARM::VST1q64LowTPseudo_UPD };
4654 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4655 ARM::VST1q16HighTPseudo_UPD,
4656 ARM::VST1q32HighTPseudo_UPD,
4657 ARM::VST1q64HighTPseudo_UPD };
4658 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4665 if (Subtarget->hasNEON()) {
4666 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4667 ARM::VST1d16QPseudoWB_fixed,
4668 ARM::VST1d32QPseudoWB_fixed,
4669 ARM::VST1d64QPseudoWB_fixed };
4670 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4671 ARM::VST1q16LowQPseudo_UPD,
4672 ARM::VST1q32LowQPseudo_UPD,
4673 ARM::VST1q64LowQPseudo_UPD };
4674 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4675 ARM::VST1q16HighQPseudo_UPD,
4676 ARM::VST1q32HighQPseudo_UPD,
4677 ARM::VST1q64HighQPseudo_UPD };
4678 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4684 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4685 ARM::VST2LNd16Pseudo_UPD,
4686 ARM::VST2LNd32Pseudo_UPD };
4687 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4688 ARM::VST2LNq32Pseudo_UPD };
4689 SelectVLDSTLane(
N,
false,
true, 2, DOpcodes, QOpcodes);
4694 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4695 ARM::VST3LNd16Pseudo_UPD,
4696 ARM::VST3LNd32Pseudo_UPD };
4697 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4698 ARM::VST3LNq32Pseudo_UPD };
4699 SelectVLDSTLane(
N,
false,
true, 3, DOpcodes, QOpcodes);
4704 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4705 ARM::VST4LNd16Pseudo_UPD,
4706 ARM::VST4LNd32Pseudo_UPD };
4707 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4708 ARM::VST4LNq32Pseudo_UPD };
4709 SelectVLDSTLane(
N,
false,
true, 4, DOpcodes, QOpcodes);
4715 unsigned IntNo =
N->getConstantOperandVal(1);
4720 case Intrinsic::arm_mrrc:
4721 case Intrinsic::arm_mrrc2: {
4726 if (Subtarget->isThumb())
4727 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4729 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4732 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(2), dl));
4733 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(3), dl));
4734 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(4), dl));
4739 if (Opc != ARM::MRRC2) {
4741 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4747 const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
4749 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4752 case Intrinsic::arm_ldaexd:
4753 case Intrinsic::arm_ldrexd: {
4756 SDValue MemAddr =
N->getOperand(2);
4757 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4759 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4760 unsigned NewOpc =
isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4761 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4764 std::vector<EVT> ResTys;
4766 ResTys.push_back(MVT::i32);
4767 ResTys.push_back(MVT::i32);
4769 ResTys.push_back(MVT::Untyped);
4770 ResTys.push_back(MVT::Other);
4774 CurDAG->getRegister(0, MVT::i32), Chain};
4775 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4778 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4788 CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
4789 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4790 dl, MVT::i32,
SDValue(Ld, 0), SubRegIdx);
4793 ReplaceUses(
SDValue(
N, 0), Result);
4801 CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
4802 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4803 dl, MVT::i32,
SDValue(Ld, 0), SubRegIdx);
4806 ReplaceUses(
SDValue(
N, 1), Result);
4808 ReplaceUses(
SDValue(
N, 2), OutChain);
4809 CurDAG->RemoveDeadNode(
N);
4812 case Intrinsic::arm_stlexd:
4813 case Intrinsic::arm_strexd: {
4818 SDValue MemAddr =
N->getOperand(4);
4822 const EVT ResTys[] = {MVT::i32, MVT::Other};
4824 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
4835 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4838 bool IsRelease = IntNo == Intrinsic::arm_stlexd;
4839 unsigned NewOpc =
isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
4840 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
4842 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4845 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4851 case Intrinsic::arm_neon_vld1: {
4852 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
4853 ARM::VLD1d32, ARM::VLD1d64 };
4854 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4855 ARM::VLD1q32, ARM::VLD1q64};
4856 SelectVLD(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
4860 case Intrinsic::arm_neon_vld1x2: {
4861 static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4862 ARM::VLD1q32, ARM::VLD1q64 };
4863 static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
4864 ARM::VLD1d16QPseudo,
4865 ARM::VLD1d32QPseudo,
4866 ARM::VLD1d64QPseudo };
4867 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4871 case Intrinsic::arm_neon_vld1x3: {
4872 static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
4873 ARM::VLD1d16TPseudo,
4874 ARM::VLD1d32TPseudo,
4875 ARM::VLD1d64TPseudo };
4876 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
4877 ARM::VLD1q16LowTPseudo_UPD,
4878 ARM::VLD1q32LowTPseudo_UPD,
4879 ARM::VLD1q64LowTPseudo_UPD };
4880 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
4881 ARM::VLD1q16HighTPseudo,
4882 ARM::VLD1q32HighTPseudo,
4883 ARM::VLD1q64HighTPseudo };
4884 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4888 case Intrinsic::arm_neon_vld1x4: {
4889 static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
4890 ARM::VLD1d16QPseudo,
4891 ARM::VLD1d32QPseudo,
4892 ARM::VLD1d64QPseudo };
4893 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
4894 ARM::VLD1q16LowQPseudo_UPD,
4895 ARM::VLD1q32LowQPseudo_UPD,
4896 ARM::VLD1q64LowQPseudo_UPD };
4897 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
4898 ARM::VLD1q16HighQPseudo,
4899 ARM::VLD1q32HighQPseudo,
4900 ARM::VLD1q64HighQPseudo };
4901 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4905 case Intrinsic::arm_neon_vld2: {
4906 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
4907 ARM::VLD2d32, ARM::VLD1q64 };
4908 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
4909 ARM::VLD2q32Pseudo };
4910 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4914 case Intrinsic::arm_neon_vld3: {
4915 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
4918 ARM::VLD1d64TPseudo };
4919 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4920 ARM::VLD3q16Pseudo_UPD,
4921 ARM::VLD3q32Pseudo_UPD };
4922 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
4923 ARM::VLD3q16oddPseudo,
4924 ARM::VLD3q32oddPseudo };
4925 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4929 case Intrinsic::arm_neon_vld4: {
4930 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
4933 ARM::VLD1d64QPseudo };
4934 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
4935 ARM::VLD4q16Pseudo_UPD,
4936 ARM::VLD4q32Pseudo_UPD };
4937 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
4938 ARM::VLD4q16oddPseudo,
4939 ARM::VLD4q32oddPseudo };
4940 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4944 case Intrinsic::arm_neon_vld2dup: {
4945 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4946 ARM::VLD2DUPd32, ARM::VLD1q64 };
4947 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4948 ARM::VLD2DUPq16EvenPseudo,
4949 ARM::VLD2DUPq32EvenPseudo };
4950 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
4951 ARM::VLD2DUPq16OddPseudo,
4952 ARM::VLD2DUPq32OddPseudo };
4953 SelectVLDDup(
N,
true,
false, 2,
4954 DOpcodes, QOpcodes0, QOpcodes1);
4958 case Intrinsic::arm_neon_vld3dup: {
4959 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
4960 ARM::VLD3DUPd16Pseudo,
4961 ARM::VLD3DUPd32Pseudo,
4962 ARM::VLD1d64TPseudo };
4963 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4964 ARM::VLD3DUPq16EvenPseudo,
4965 ARM::VLD3DUPq32EvenPseudo };
4966 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
4967 ARM::VLD3DUPq16OddPseudo,
4968 ARM::VLD3DUPq32OddPseudo };
4969 SelectVLDDup(
N,
true,
false, 3,
4970 DOpcodes, QOpcodes0, QOpcodes1);
4974 case Intrinsic::arm_neon_vld4dup: {
4975 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
4976 ARM::VLD4DUPd16Pseudo,
4977 ARM::VLD4DUPd32Pseudo,
4978 ARM::VLD1d64QPseudo };
4979 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4980 ARM::VLD4DUPq16EvenPseudo,
4981 ARM::VLD4DUPq32EvenPseudo };
4982 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
4983 ARM::VLD4DUPq16OddPseudo,
4984 ARM::VLD4DUPq32OddPseudo };
4985 SelectVLDDup(
N,
true,
false, 4,
4986 DOpcodes, QOpcodes0, QOpcodes1);
4990 case Intrinsic::arm_neon_vld2lane: {
4991 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
4992 ARM::VLD2LNd16Pseudo,
4993 ARM::VLD2LNd32Pseudo };
4994 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
4995 ARM::VLD2LNq32Pseudo };
4996 SelectVLDSTLane(
N,
true,
false, 2, DOpcodes, QOpcodes);
5000 case Intrinsic::arm_neon_vld3lane: {
5001 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
5002 ARM::VLD3LNd16Pseudo,
5003 ARM::VLD3LNd32Pseudo };
5004 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
5005 ARM::VLD3LNq32Pseudo };
5006 SelectVLDSTLane(
N,
true,
false, 3, DOpcodes, QOpcodes);
5010 case Intrinsic::arm_neon_vld4lane: {
5011 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
5012 ARM::VLD4LNd16Pseudo,
5013 ARM::VLD4LNd32Pseudo };
5014 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
5015 ARM::VLD4LNq32Pseudo };
5016 SelectVLDSTLane(
N,
true,
false, 4, DOpcodes, QOpcodes);
5020 case Intrinsic::arm_neon_vst1: {
5021 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
5022 ARM::VST1d32, ARM::VST1d64 };
5023 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5024 ARM::VST1q32, ARM::VST1q64 };
5025 SelectVST(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
5029 case Intrinsic::arm_neon_vst1x2: {
5030 static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5031 ARM::VST1q32, ARM::VST1q64 };
5032 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
5033 ARM::VST1d16QPseudo,
5034 ARM::VST1d32QPseudo,
5035 ARM::VST1d64QPseudo };
5036 SelectVST(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
5040 case Intrinsic::arm_neon_vst1x3: {
5041 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
5042 ARM::VST1d16TPseudo,
5043 ARM::VST1d32TPseudo,
5044 ARM::VST1d64TPseudo };
5045 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
5046 ARM::VST1q16LowTPseudo_UPD,
5047 ARM::VST1q32LowTPseudo_UPD,
5048 ARM::VST1q64LowTPseudo_UPD };
5049 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
5050 ARM::VST1q16HighTPseudo,
5051 ARM::VST1q32HighTPseudo,
5052 ARM::VST1q64HighTPseudo };
5053 SelectVST(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5057 case Intrinsic::arm_neon_vst1x4: {
5058 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
5059 ARM::VST1d16QPseudo,
5060 ARM::VST1d32QPseudo,
5061 ARM::VST1d64QPseudo };
5062 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
5063 ARM::VST1q16LowQPseudo_UPD,
5064 ARM::VST1q32LowQPseudo_UPD,
5065 ARM::VST1q64LowQPseudo_UPD };
5066 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
5067 ARM::VST1q16HighQPseudo,
5068 ARM::VST1q32HighQPseudo,
5069 ARM::VST1q64HighQPseudo };
5070 SelectVST(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5074 case Intrinsic::arm_neon_vst2: {
5075 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
5076 ARM::VST2d32, ARM::VST1q64 };
5077 static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
5078 ARM::VST2q32Pseudo };
5079 SelectVST(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
5083 case Intrinsic::arm_neon_vst3: {
5084 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
5087 ARM::VST1d64TPseudo };
5088 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
5089 ARM::VST3q16Pseudo_UPD,
5090 ARM::VST3q32Pseudo_UPD };
5091 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
5092 ARM::VST3q16oddPseudo,
5093 ARM::VST3q32oddPseudo };
5094 SelectVST(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5098 case Intrinsic::arm_neon_vst4: {
5099 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
5102 ARM::VST1d64QPseudo };
5103 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
5104 ARM::VST4q16Pseudo_UPD,
5105 ARM::VST4q32Pseudo_UPD };
5106 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
5107 ARM::VST4q16oddPseudo,
5108 ARM::VST4q32oddPseudo };
5109 SelectVST(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5113 case Intrinsic::arm_neon_vst2lane: {
5114 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
5115 ARM::VST2LNd16Pseudo,
5116 ARM::VST2LNd32Pseudo };
5117 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
5118 ARM::VST2LNq32Pseudo };
5119 SelectVLDSTLane(
N,
false,
false, 2, DOpcodes, QOpcodes);
5123 case Intrinsic::arm_neon_vst3lane: {
5124 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
5125 ARM::VST3LNd16Pseudo,
5126 ARM::VST3LNd32Pseudo };
5127 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
5128 ARM::VST3LNq32Pseudo };
5129 SelectVLDSTLane(
N,
false,
false, 3, DOpcodes, QOpcodes);
5133 case Intrinsic::arm_neon_vst4lane: {
5134 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
5135 ARM::VST4LNd16Pseudo,
5136 ARM::VST4LNd32Pseudo };
5137 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
5138 ARM::VST4LNq32Pseudo };
5139 SelectVLDSTLane(
N,
false,
false, 4, DOpcodes, QOpcodes);
5143 case Intrinsic::arm_mve_vldr_gather_base_wb:
5144 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
5145 static const uint16_t Opcodes[] = {ARM::MVE_VLDRWU32_qi_pre,
5146 ARM::MVE_VLDRDU64_qi_pre};
5147 SelectMVE_WB(
N, Opcodes,
5148 IntNo == Intrinsic::arm_mve_vldr_gather_base_wb_predicated);
5152 case Intrinsic::arm_mve_vld2q: {
5153 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8, ARM::MVE_VLD21_8};
5154 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
5156 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
5158 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5159 SelectMVE_VLD(
N, 2, Opcodes,
false);
5163 case Intrinsic::arm_mve_vld4q: {
5164 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
5165 ARM::MVE_VLD42_8, ARM::MVE_VLD43_8};
5166 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
5169 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
5172 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5173 SelectMVE_VLD(
N, 4, Opcodes,
false);
5181 unsigned IntNo =
N->getConstantOperandVal(0);
5187 case Intrinsic::arm_neon_vcvtbfp2bf: {
5189 const SDValue &Src =
N->getOperand(1);
5192 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5193 SDValue Ops[] = { Src, Src, Pred, Reg0 };
5194 CurDAG->SelectNodeTo(
N, ARM::BF16_VCVTB, DestTy, Ops);
5199 case Intrinsic::arm_neon_vcvtfp2bf: {
5201 const SDValue &Src =
N->getOperand(1);
5203 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5204 SDValue Ops[] = { Src, Pred, Reg0 };
5205 CurDAG->SelectNodeTo(
N, ARM::BF16_VCVT, MVT::v4bf16, Ops);
5209 case Intrinsic::arm_mve_urshrl:
5210 SelectMVE_LongShift(
N, ARM::MVE_URSHRL,
true,
false);
5212 case Intrinsic::arm_mve_uqshll:
5213 SelectMVE_LongShift(
N, ARM::MVE_UQSHLL,
true,
false);
5215 case Intrinsic::arm_mve_srshrl:
5216 SelectMVE_LongShift(
N, ARM::MVE_SRSHRL,
true,
false);
5218 case Intrinsic::arm_mve_sqshll:
5219 SelectMVE_LongShift(
N, ARM::MVE_SQSHLL,
true,
false);
5221 case Intrinsic::arm_mve_uqrshll:
5222 SelectMVE_LongShift(
N, ARM::MVE_UQRSHLL,
false,
true);
5224 case Intrinsic::arm_mve_sqrshrl:
5225 SelectMVE_LongShift(
N, ARM::MVE_SQRSHRL,
false,
true);
5228 case Intrinsic::arm_mve_vadc:
5229 case Intrinsic::arm_mve_vadc_predicated:
5230 SelectMVE_VADCSBC(
N, ARM::MVE_VADC, ARM::MVE_VADCI,
true,
5231 IntNo == Intrinsic::arm_mve_vadc_predicated);
5233 case Intrinsic::arm_mve_vsbc:
5234 case Intrinsic::arm_mve_vsbc_predicated:
5235 SelectMVE_VADCSBC(
N, ARM::MVE_VSBC, ARM::MVE_VSBCI,
true,
5236 IntNo == Intrinsic::arm_mve_vsbc_predicated);
5238 case Intrinsic::arm_mve_vshlc:
5239 case Intrinsic::arm_mve_vshlc_predicated:
5240 SelectMVE_VSHLC(
N, IntNo == Intrinsic::arm_mve_vshlc_predicated);
5243 case Intrinsic::arm_mve_vmlldava:
5244 case Intrinsic::arm_mve_vmlldava_predicated: {
5245 static const uint16_t OpcodesU[] = {
5246 ARM::MVE_VMLALDAVu16, ARM::MVE_VMLALDAVu32,
5247 ARM::MVE_VMLALDAVau16, ARM::MVE_VMLALDAVau32,
5249 static const uint16_t OpcodesS[] = {
5250 ARM::MVE_VMLALDAVs16, ARM::MVE_VMLALDAVs32,
5251 ARM::MVE_VMLALDAVas16, ARM::MVE_VMLALDAVas32,
5252 ARM::MVE_VMLALDAVxs16, ARM::MVE_VMLALDAVxs32,
5253 ARM::MVE_VMLALDAVaxs16, ARM::MVE_VMLALDAVaxs32,
5254 ARM::MVE_VMLSLDAVs16, ARM::MVE_VMLSLDAVs32,
5255 ARM::MVE_VMLSLDAVas16, ARM::MVE_VMLSLDAVas32,
5256 ARM::MVE_VMLSLDAVxs16, ARM::MVE_VMLSLDAVxs32,
5257 ARM::MVE_VMLSLDAVaxs16, ARM::MVE_VMLSLDAVaxs32,
5259 SelectMVE_VMLLDAV(
N, IntNo == Intrinsic::arm_mve_vmlldava_predicated,
5260 OpcodesS, OpcodesU);
5264 case Intrinsic::arm_mve_vrmlldavha:
5265 case Intrinsic::arm_mve_vrmlldavha_predicated: {
5266 static const uint16_t OpcodesU[] = {
5267 ARM::MVE_VRMLALDAVHu32, ARM::MVE_VRMLALDAVHau32,
5269 static const uint16_t OpcodesS[] = {
5270 ARM::MVE_VRMLALDAVHs32, ARM::MVE_VRMLALDAVHas32,
5271 ARM::MVE_VRMLALDAVHxs32, ARM::MVE_VRMLALDAVHaxs32,
5272 ARM::MVE_VRMLSLDAVHs32, ARM::MVE_VRMLSLDAVHas32,
5273 ARM::MVE_VRMLSLDAVHxs32, ARM::MVE_VRMLSLDAVHaxs32,
5275 SelectMVE_VRMLLDAVH(
N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated,
5276 OpcodesS, OpcodesU);
5280 case Intrinsic::arm_mve_vidup:
5281 case Intrinsic::arm_mve_vidup_predicated: {
5282 static const uint16_t Opcodes[] = {
5283 ARM::MVE_VIDUPu8, ARM::MVE_VIDUPu16, ARM::MVE_VIDUPu32,
5285 SelectMVE_VxDUP(
N, Opcodes,
false,
5286 IntNo == Intrinsic::arm_mve_vidup_predicated);
5290 case Intrinsic::arm_mve_vddup:
5291 case Intrinsic::arm_mve_vddup_predicated: {
5292 static const uint16_t Opcodes[] = {
5293 ARM::MVE_VDDUPu8, ARM::MVE_VDDUPu16, ARM::MVE_VDDUPu32,
5295 SelectMVE_VxDUP(
N, Opcodes,
false,
5296 IntNo == Intrinsic::arm_mve_vddup_predicated);
5300 case Intrinsic::arm_mve_viwdup:
5301 case Intrinsic::arm_mve_viwdup_predicated: {
5302 static const uint16_t Opcodes[] = {
5303 ARM::MVE_VIWDUPu8, ARM::MVE_VIWDUPu16, ARM::MVE_VIWDUPu32,
5305 SelectMVE_VxDUP(
N, Opcodes,
true,
5306 IntNo == Intrinsic::arm_mve_viwdup_predicated);
5310 case Intrinsic::arm_mve_vdwdup:
5311 case Intrinsic::arm_mve_vdwdup_predicated: {
5312 static const uint16_t Opcodes[] = {
5313 ARM::MVE_VDWDUPu8, ARM::MVE_VDWDUPu16, ARM::MVE_VDWDUPu32,
5315 SelectMVE_VxDUP(
N, Opcodes,
true,
5316 IntNo == Intrinsic::arm_mve_vdwdup_predicated);
5320 case Intrinsic::arm_cde_cx1d:
5321 case Intrinsic::arm_cde_cx1da:
5322 case Intrinsic::arm_cde_cx2d:
5323 case Intrinsic::arm_cde_cx2da:
5324 case Intrinsic::arm_cde_cx3d:
5325 case Intrinsic::arm_cde_cx3da: {
5326 bool HasAccum = IntNo == Intrinsic::arm_cde_cx1da ||
5327 IntNo == Intrinsic::arm_cde_cx2da ||
5328 IntNo == Intrinsic::arm_cde_cx3da;
5332 case Intrinsic::arm_cde_cx1d:
5333 case Intrinsic::arm_cde_cx1da:
5335 Opcode = HasAccum ? ARM::CDE_CX1DA : ARM::CDE_CX1D;
5337 case Intrinsic::arm_cde_cx2d:
5338 case Intrinsic::arm_cde_cx2da:
5340 Opcode = HasAccum ? ARM::CDE_CX2DA : ARM::CDE_CX2D;
5342 case Intrinsic::arm_cde_cx3d:
5343 case Intrinsic::arm_cde_cx3da:
5345 Opcode = HasAccum ? ARM::CDE_CX3DA : ARM::CDE_CX3D;
5350 SelectCDE_CXxD(
N, Opcode, NumExtraOps, HasAccum);
5373 std::vector<SDValue> &Ops) {
5375 RegString.
split(Fields,
':');
5377 if (Fields.
size() > 1) {
5378 bool AllIntFields =
true;
5383 AllIntFields &= !
Field.trim(
"CPcp").getAsInteger(10, IntField);
5388 "Unexpected non-integer value in special register string.");
5399 auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.
lower());
5402 return TheReg->Encoding;
5414 .
Case(
"nzcvqg", 0x3)
5422 auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
5423 const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
5424 if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
5426 return (
int)(TheReg->Encoding & 0xFFF);
5435 if (Reg ==
"apsr") {
5445 if (Reg !=
"cpsr" && Reg !=
"spsr") {
5450 if (Flags.empty() || Flags ==
"all")
5455 for (
char Flag : Flags) {
5475 if (!FlagVal || (Mask & FlagVal))
5490bool ARMDAGToDAGISel::tryReadRegister(
SDNode *
N){
5491 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
5492 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5493 bool IsThumb2 = Subtarget->
isThumb2();
5496 std::vector<SDValue> Ops;
5506 if (Ops.size() == 5){
5507 Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
5508 ResTypes.
append({ MVT::i32, MVT::Other });
5510 assert(Ops.size() == 3 &&
5511 "Invalid number of fields in special register string.");
5512 Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
5513 ResTypes.
append({ MVT::i32, MVT::i32, MVT::Other });
5516 Ops.push_back(
getAL(CurDAG,
DL));
5517 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5518 Ops.push_back(
N->getOperand(0));
5519 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, ResTypes, Ops));
5523 std::string SpecialReg = RegString->getString().lower();
5526 if (BankedReg != -1) {
5527 Ops = { CurDAG->getTargetConstant(BankedReg,
DL, MVT::i32),
5528 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5531 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
5532 DL, MVT::i32, MVT::Other, Ops));
5540 .
Case(
"fpscr", ARM::VMRS)
5541 .
Case(
"fpexc", ARM::VMRS_FPEXC)
5542 .
Case(
"fpsid", ARM::VMRS_FPSID)
5543 .
Case(
"mvfr0", ARM::VMRS_MVFR0)
5544 .
Case(
"mvfr1", ARM::VMRS_MVFR1)
5545 .
Case(
"mvfr2", ARM::VMRS_MVFR2)
5546 .
Case(
"fpinst", ARM::VMRS_FPINST)
5547 .
Case(
"fpinst2", ARM::VMRS_FPINST2)
5557 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5560 CurDAG->getMachineNode(Opcode,
DL, MVT::i32, MVT::Other, Ops));
5569 if (SYSmValue == -1)
5572 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue,
DL, MVT::i32),
5573 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5576 N, CurDAG->getMachineNode(ARM::t2MRS_M,
DL, MVT::i32, MVT::Other, Ops));
5582 if (SpecialReg ==
"apsr" || SpecialReg ==
"cpsr") {
5583 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5585 ReplaceNode(
N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
5586 DL, MVT::i32, MVT::Other, Ops));
5590 if (SpecialReg ==
"spsr") {
5591 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5594 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys,
DL,
5595 MVT::i32, MVT::Other, Ops));
5605bool ARMDAGToDAGISel::tryWriteRegister(
SDNode *
N){
5606 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
5607 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5608 bool IsThumb2 = Subtarget->
isThumb2();
5611 std::vector<SDValue> Ops;
5620 if (Ops.size() == 5) {
5621 Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
5622 Ops.insert(Ops.begin()+2,
N->getOperand(2));
5624 assert(Ops.size() == 3 &&
5625 "Invalid number of fields in special register string.");
5626 Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
5627 SDValue WriteValue[] = {
N->getOperand(2),
N->getOperand(3) };
5628 Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
5631 Ops.push_back(
getAL(CurDAG,
DL));
5632 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5633 Ops.push_back(
N->getOperand(0));
5635 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
5639 std::string SpecialReg = RegString->getString().lower();
5641 if (BankedReg != -1) {
5642 Ops = { CurDAG->getTargetConstant(BankedReg,
DL, MVT::i32),
N->getOperand(2),
5643 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5646 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
5647 DL, MVT::Other, Ops));
5655 .
Case(
"fpscr", ARM::VMSR)
5656 .
Case(
"fpexc", ARM::VMSR_FPEXC)
5657 .
Case(
"fpsid", ARM::VMSR_FPSID)
5658 .
Case(
"fpinst", ARM::VMSR_FPINST)
5659 .
Case(
"fpinst2", ARM::VMSR_FPINST2)
5665 Ops = {
N->getOperand(2),
getAL(CurDAG,
DL),
5666 CurDAG->getRegister(0, MVT::i32),
N->getOperand(0) };
5667 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
5671 std::pair<StringRef, StringRef> Fields;
5673 std::string
Reg = Fields.first.str();
5680 if (SYSmValue == -1)
5683 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue,
DL, MVT::i32),
5684 N->getOperand(2),
getAL(CurDAG,
DL),
5685 CurDAG->getRegister(0, MVT::i32),
N->getOperand(0) };
5686 ReplaceNode(
N, CurDAG->getMachineNode(ARM::t2MSR_M,
DL, MVT::Other, Ops));
5695 Ops = { CurDAG->getTargetConstant(Mask,
DL, MVT::i32),
N->getOperand(2),
5696 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5698 ReplaceNode(
N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
5699 DL, MVT::Other, Ops));
5706bool ARMDAGToDAGISel::tryInlineAsm(
SDNode *
N){
5707 std::vector<SDValue> AsmNodeOperands;
5709 bool Changed =
false;
5710 unsigned NumOps =
N->getNumOperands();
5725 for(
unsigned i = 0, e =
N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
5727 AsmNodeOperands.push_back(
op);
5732 if (
const auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(i)))
5741 if (
Flag.isImmKind()) {
5743 AsmNodeOperands.push_back(
op);
5747 const unsigned NumRegs =
Flag.getNumOperandRegisters();
5751 unsigned DefIdx = 0;
5752 bool IsTiedToChangedOp =
false;
5755 if (Changed &&
Flag.isUseOperandTiedToDef(DefIdx))
5756 IsTiedToChangedOp = OpChanged[DefIdx];
5763 if (
Flag.isMemKind()) {
5765 AsmNodeOperands.push_back(
op);
5769 if (!
Flag.isRegUseKind() && !
Flag.isRegDefKind() &&
5770 !
Flag.isRegDefEarlyClobberKind())
5774 const bool HasRC =
Flag.hasRegClassConstraint(RC);
5775 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
5779 assert((i+2 < NumOps) &&
"Invalid number of operands in inline asm");
5782 Register Reg0 = cast<RegisterSDNode>(V0)->getReg();
5783 Register Reg1 = cast<RegisterSDNode>(V1)->getReg();
5787 if (
Flag.isRegDefKind() ||
Flag.isRegDefEarlyClobberKind()) {
5791 Register GPVR =
MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5792 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5795 SDNode *GU =
N->getGluedUser();
5796 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
5800 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
5802 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
5804 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
5810 Ops.push_back(
T1.getValue(1));
5811 CurDAG->UpdateNodeOperands(GU, Ops);
5818 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
5820 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
5826 Register GPVR =
MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5827 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5828 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair,
T1.getValue(1));
5837 OpChanged[OpChanged.
size() -1 ] =
true;
5839 if (IsTiedToChangedOp)
5840 Flag.setMatchingOp(DefIdx);
5842 Flag.setRegClass(ARM::GPRPairRegClassID);
5844 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
5845 Flag, dl, MVT::i32);
5847 AsmNodeOperands.push_back(PairedReg);
5854 AsmNodeOperands.push_back(Glue);
5859 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
5861 ReplaceNode(
N,
New.getNode());
5865bool ARMDAGToDAGISel::SelectInlineAsmMemoryOperand(
5867 std::vector<SDValue> &OutOps) {
5868 switch(ConstraintID) {
5871 case InlineAsm::ConstraintCode::m:
5872 case InlineAsm::ConstraintCode::o:
5873 case InlineAsm::ConstraintCode::Q:
5874 case InlineAsm::ConstraintCode::Um:
5875 case InlineAsm::ConstraintCode::Un:
5876 case InlineAsm::ConstraintCode::Uq:
5877 case InlineAsm::ConstraintCode::Us:
5878 case InlineAsm::ConstraintCode::Ut:
5879 case InlineAsm::ConstraintCode::Uv:
5880 case InlineAsm::ConstraintCode::Uy:
5884 OutOps.push_back(
Op);
5895 return new ARMDAGToDAGISelLegacy(
TM, OptLevel);
unsigned const MachineRegisterInfo * MRI
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
static bool isThumb(const MCSubtargetInfo &STI)
static unsigned getVectorShuffleOpcode(EVT VT, unsigned Opc64[3], unsigned Opc128[3])
static int getBankedRegisterMask(StringRef RegString)
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
static bool isVSTfixed(unsigned Opc)
static bool isVLDfixed(unsigned Opc)
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static std::optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
static bool shouldUseZeroOffsetLdSt(SDValue N)
static int getMClassFlagsMask(StringRef Flags)
static bool SDValueToConstBool(SDValue SDVal)
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
This file describes how to lower LLVM code to machine code.
support::ulittle16_t & Lo
support::ulittle16_t & Hi
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
An arbitrary precision integer that knows its signedness.
bool isThumb1Only() const
bool hasFPARMv8Base() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
uint64_t getZExtValue() const
This class represents an Operation in the Expression.
Container class for subtarget features.
FunctionPass class - This class is used to implement most global optimizations.
This class is used to form a handle around another node that is persistent and is updated across invo...
Base class for LoadSDNode and StoreSDNode.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
unsigned getOpcode() const
Return the opcode number for this descriptor.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MLOAD node.
This is an abstract virtual class for memory operations.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
int getNodeId() const
Return the unique node id.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string lower() const
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt32Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
float getFPImmFloat(unsigned Imm)
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOptLevel OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
CodeGenOptLevel
Code generation optimization level.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ And
Bitwise or logical AND of integers.
@ NearestTiesToEven
roundTiesToEven.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
These are IR-level optimization flags that may be propagated to SDNodes.