32#include "llvm/IR/IntrinsicsARM.h"
42#define DEBUG_TYPE "arm-isel"
43#define PASS_NAME "ARM Instruction Selection"
47 cl::desc(
"Disable isel of shifter-op"),
64 ARMDAGToDAGISel() =
delete;
80 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &dl) {
81 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
90 bool hasNoVMLxHazardUse(
SDNode *
N)
const;
91 bool isShifterOpProfitable(
const SDValue &Shift,
95 bool CheckProfitability =
true);
97 SDValue &
B,
bool CheckProfitability =
true);
101 return SelectRegShifterOperand(
N,
A,
B,
C,
false);
105 return SelectImmShifterOperand(
N,
A,
B,
false);
110 return SelectImmShifterOperand(
N,
A,
B,
false);
121 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
156 template <
unsigned Shift>
161 template <
unsigned Shift>
167 template <
unsigned Shift>
171 template <
unsigned Shift>
177 template<
int Min,
int Max>
180 inline bool is_so_imm(
unsigned Imm)
const {
184 inline bool is_so_imm_not(
unsigned Imm)
const {
188 inline bool is_t2_so_imm(
unsigned Imm)
const {
192 inline bool is_t2_so_imm_not(
unsigned Imm)
const {
197#include "ARMGenDAGISel.inc"
203 bool tryARMIndexedLoad(
SDNode *
N);
204 bool tryT1IndexedLoad(
SDNode *
N);
205 bool tryT2IndexedLoad(
SDNode *
N);
206 bool tryMVEIndexedLoad(
SDNode *
N);
217 void SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
225 void SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
232 void SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
233 unsigned NumVecs,
const uint16_t *DOpcodes,
237 template <
typename SDValueVector>
238 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
240 template <
typename SDValueVector>
241 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
244 template <
typename SDValueVector>
245 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc);
246 template <
typename SDValueVector>
247 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
EVT InactiveTy);
250 void SelectMVE_WB(
SDNode *
N,
const uint16_t *Opcodes,
bool Predicated);
254 bool HasSaturationOperand);
258 uint16_t OpcodeWithNoCarry,
bool Add,
bool Predicated);
262 void SelectMVE_VSHLC(
SDNode *
N,
bool Predicated);
272 void SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
274 size_t Stride,
size_t TySize);
278 void SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
282 void SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
290 void SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
291 const uint16_t *
const *Opcodes,
bool HasWriteback);
296 bool Wrapping,
bool Predicated);
310 void SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
bool isUpdating,
311 unsigned NumVecs,
const uint16_t *DOpcodes,
312 const uint16_t *QOpcodes0 =
nullptr,
313 const uint16_t *QOpcodes1 =
nullptr);
318 bool tryInsertVectorElt(
SDNode *
N);
323 bool tryReadRegister(
SDNode *
N);
324 bool tryWriteRegister(
SDNode *
N);
328 void SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI);
330 void SelectCMP_SWAP(
SDNode *
N);
336 std::vector<SDValue> &OutOps)
override;
358 bool canExtractShiftFromMul(
const SDValue &
N,
unsigned MaxShift,
359 unsigned &PowerOfTwo,
SDValue &NewMulConst)
const;
367char ARMDAGToDAGISel::ID = 0;
374 if (
N->getOpcode() ==
ISD::Constant &&
N->getValueType(0) == MVT::i32) {
375 Imm =
N->getAsZExtVal();
391 return N->getOpcode() == Opc &&
400 int RangeMin,
int RangeMax,
401 int &ScaledConstant) {
402 assert(Scale > 0 &&
"Invalid scale!");
409 ScaledConstant = (int)
C->getZExtValue();
410 if ((ScaledConstant % Scale) != 0)
413 ScaledConstant /= Scale;
414 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
417void ARMDAGToDAGISel::PreprocessISelDAG() {
418 if (!Subtarget->hasV6T2Ops())
421 bool isThumb2 = Subtarget->isThumb();
438 unsigned And_imm = 0;
448 if (TZ != 1 && TZ != 2)
460 if (And_imm & (And_imm + 1))
465 unsigned Srl_imm = 0;
476 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
479 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
480 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
487 CurDAG->getConstant(Srl_imm + TZ,
SDLoc(Srl),
491 CurDAG->getConstant(And_imm,
SDLoc(Srl), MVT::i32));
493 N1, CurDAG->getConstant(TZ,
SDLoc(Srl), MVT::i32));
494 CurDAG->UpdateNodeOperands(&
N, N0, N1);
501bool ARMDAGToDAGISel::hasNoVMLxHazardUse(
SDNode *
N)
const {
502 if (OptLevel == CodeGenOptLevel::None)
505 if (!Subtarget->hasVMLxHazards())
514 if (
Use->isMachineOpcode()) {
516 CurDAG->getSubtarget().getInstrInfo());
522 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
537 return TII->isFpMLxInstruction(Opcode);
543bool ARMDAGToDAGISel::isShifterOpProfitable(
const SDValue &Shift,
552 (ShAmt == 2 || (Subtarget->
isSwift() && ShAmt == 1));
555bool ARMDAGToDAGISel::canExtractShiftFromMul(
const SDValue &
N,
557 unsigned &PowerOfTwo,
564 if (!
N.hasOneUse())
return false;
567 if (!MulConst)
return false;
570 if (!MulConst->
hasOneUse())
return false;
572 if (MulConstVal == 0)
return false;
575 PowerOfTwo = MaxShift;
576 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
578 if (PowerOfTwo == 0)
return false;
582 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
583 NewMulConst = CurDAG->getConstant(NewMulConstVal,
SDLoc(
N), MVT::i32);
586 return NewCost < OldCost;
590 CurDAG->RepositionNode(
N.getNode()->getIterator(),
M.getNode());
594bool ARMDAGToDAGISel::SelectImmShifterOperand(
SDValue N,
597 bool CheckProfitability) {
604 unsigned PowerOfTwo = 0;
606 if (canExtractShiftFromMul(
N, 31, PowerOfTwo, NewMulConst)) {
609 replaceDAGValue(
N.getOperand(1), NewMulConst);
611 Opc = CurDAG->getTargetConstant(
623 BaseReg =
N.getOperand(0);
624 unsigned ShImmVal = 0;
626 if (!RHS)
return false;
627 ShImmVal =
RHS->getZExtValue() & 31;
633bool ARMDAGToDAGISel::SelectRegShifterOperand(
SDValue N,
637 bool CheckProfitability) {
647 BaseReg =
N.getOperand(0);
648 unsigned ShImmVal = 0;
650 if (RHS)
return false;
652 ShReg =
N.getOperand(1);
653 if (CheckProfitability && !isShifterOpProfitable(
N, ShOpcVal, ShImmVal))
665 return CurDAG->haveNoCommonBitsSet(
N, Parent->
getOperand(1));
669bool ARMDAGToDAGISel::SelectAddrModeImm12(
SDValue N,
676 !CurDAG->isBaseWithConstantOffset(
N)) {
679 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
680 Base = CurDAG->getTargetFrameIndex(
681 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
682 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
690 Base =
N.getOperand(0);
693 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
698 int RHSC = (int)
RHS->getSExtValue();
702 if (RHSC > -0x1000 && RHSC < 0x1000) {
703 Base =
N.getOperand(0);
705 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
706 Base = CurDAG->getTargetFrameIndex(
707 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
709 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
716 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
728 int RHSC = (int)
RHS->getZExtValue();
737 unsigned ShAmt =
Log2_32(RHSC);
750 !CurDAG->isBaseWithConstantOffset(
N))
757 -0x1000+1, 0x1000, RHSC))
767 Base =
N.getOperand(0);
774 dyn_cast<ConstantSDNode>(
N.getOperand(1).getOperand(1))) {
775 ShAmt = Sh->getZExtValue();
776 if (isShifterOpProfitable(
Offset, ShOpcVal, ShAmt))
777 Offset =
N.getOperand(1).getOperand(0);
790 N.getOperand(0).hasOneUse())) {
796 dyn_cast<ConstantSDNode>(
N.getOperand(0).getOperand(1))) {
797 ShAmt = Sh->getZExtValue();
798 if (isShifterOpProfitable(
N.getOperand(0), ShOpcVal, ShAmt)) {
799 Offset =
N.getOperand(0).getOperand(0);
800 Base =
N.getOperand(1);
814 unsigned PowerOfTwo = 0;
816 if (canExtractShiftFromMul(
Offset, 31, PowerOfTwo, NewMulConst)) {
818 replaceDAGValue(
Offset.getOperand(1), NewMulConst);
819 Offset = Handle.getValue();
832 unsigned Opcode =
Op->getOpcode();
834 ? cast<LoadSDNode>(
Op)->getAddressingMode()
835 : cast<StoreSDNode>(
Op)->getAddressingMode();
849 ShAmt = Sh->getZExtValue();
850 if (isShifterOpProfitable(
N, ShOpcVal, ShAmt))
868 unsigned Opcode =
Op->getOpcode();
870 ? cast<LoadSDNode>(
Op)->getAddressingMode()
871 : cast<StoreSDNode>(
Op)->getAddressingMode();
877 Offset = CurDAG->getRegister(0, MVT::i32);
878 Opc = CurDAG->getTargetConstant(Val,
SDLoc(
Op), MVT::i32);
888 unsigned Opcode =
Op->getOpcode();
890 ? cast<LoadSDNode>(
Op)->getAddressingMode()
891 : cast<StoreSDNode>(
Op)->getAddressingMode();
896 Offset = CurDAG->getRegister(0, MVT::i32);
911bool ARMDAGToDAGISel::SelectAddrMode3(
SDValue N,
916 Base =
N.getOperand(0);
923 if (!CurDAG->isBaseWithConstantOffset(
N)) {
926 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
927 Base = CurDAG->getTargetFrameIndex(
928 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
930 Offset = CurDAG->getRegister(0, MVT::i32);
939 -256 + 1, 256, RHSC)) {
940 Base =
N.getOperand(0);
942 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
943 Base = CurDAG->getTargetFrameIndex(
944 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
946 Offset = CurDAG->getRegister(0, MVT::i32);
958 Base =
N.getOperand(0);
967 unsigned Opcode =
Op->getOpcode();
969 ? cast<LoadSDNode>(
Op)->getAddressingMode()
970 : cast<StoreSDNode>(
Op)->getAddressingMode();
975 Offset = CurDAG->getRegister(0, MVT::i32);
989 if (!CurDAG->isBaseWithConstantOffset(
N)) {
992 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
993 Base = CurDAG->getTargetFrameIndex(
994 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
999 Base =
N.getOperand(0);
1008 const int Scale =
FP16 ? 2 : 4;
1011 Base =
N.getOperand(0);
1013 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1014 Base = CurDAG->getTargetFrameIndex(
1015 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1046bool ARMDAGToDAGISel::SelectAddrMode5(
SDValue N,
1051bool ARMDAGToDAGISel::SelectAddrMode5FP16(
SDValue N,
1060 unsigned Alignment = 0;
1062 MemSDNode *MemN = cast<MemSDNode>(Parent);
1064 if (isa<LSBaseSDNode>(MemN) ||
1072 if (MMOAlign.
value() >= MemSize && MemSize > 1)
1073 Alignment = MemSize;
1081 Align = CurDAG->getTargetConstant(Alignment,
SDLoc(
N), MVT::i32);
1094 Offset = CurDAG->getRegister(0, MVT::i32);
1099bool ARMDAGToDAGISel::SelectAddrModePC(
SDValue N,
1124 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
1125 return C->getSExtValue() < 0 &&
C->getSExtValue() >= -255;
1132 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N)) {
1140 Base =
N.getOperand(0);
1149 return SelectThumbAddrModeRRSext(
N,
Base,
Offset);
1153ARMDAGToDAGISel::SelectThumbAddrModeImm5S(
SDValue N,
unsigned Scale,
1157 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1161 if (!CurDAG->isBaseWithConstantOffset(
N)) {
1169 Base =
N.getOperand(0);
1174 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1181 Base =
N.getOperand(0);
1182 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1193 return SelectThumbAddrModeImm5S(
N, 4,
Base, OffImm);
1199 return SelectThumbAddrModeImm5S(
N, 2,
Base, OffImm);
1205 return SelectThumbAddrModeImm5S(
N, 1,
Base, OffImm);
1208bool ARMDAGToDAGISel::SelectThumbAddrModeSP(
SDValue N,
1211 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1217 Base = CurDAG->getTargetFrameIndex(
1218 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1219 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1223 if (!CurDAG->isBaseWithConstantOffset(
N))
1230 Base =
N.getOperand(0);
1231 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1242 Base = CurDAG->getTargetFrameIndex(
1243 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1244 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1254template <
unsigned Shift>
1257 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1261 Base =
N.getOperand(0);
1265 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1272 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1282bool ARMDAGToDAGISel::SelectT2AddrModeImm12(
SDValue N,
1288 !CurDAG->isBaseWithConstantOffset(
N)) {
1291 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1292 Base = CurDAG->getTargetFrameIndex(
1293 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1294 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1302 Base =
N.getOperand(0);
1307 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1311 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1312 if (SelectT2AddrModeImm8(
N,
Base, OffImm))
1316 int RHSC = (int)
RHS->getZExtValue();
1320 if (RHSC >= 0 && RHSC < 0x1000) {
1321 Base =
N.getOperand(0);
1323 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1324 Base = CurDAG->getTargetFrameIndex(
1325 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1327 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1334 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1338template <
unsigned Shift>
1341 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1344 Base =
N.getOperand(0);
1346 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1347 Base = CurDAG->getTargetFrameIndex(
1348 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1354 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1361 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1365bool ARMDAGToDAGISel::SelectT2AddrModeImm8(
SDValue N,
1369 !CurDAG->isBaseWithConstantOffset(
N))
1372 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1373 int RHSC = (int)
RHS->getSExtValue();
1377 if ((RHSC >= -255) && (RHSC < 0)) {
1378 Base =
N.getOperand(0);
1380 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1381 Base = CurDAG->getTargetFrameIndex(
1382 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1384 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32);
1394 unsigned Opcode =
Op->getOpcode();
1396 ? cast<LoadSDNode>(
Op)->getAddressingMode()
1397 : cast<StoreSDNode>(
Op)->getAddressingMode();
1401 ? CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i32)
1402 : CurDAG->getTargetConstant(-RHSC,
SDLoc(
N), MVT::i32);
1409template <
unsigned Shift>
1412 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1416 Base =
N.getOperand(0);
1418 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1419 Base = CurDAG->getTargetFrameIndex(
1420 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1426 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32);
1433 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1437template <
unsigned Shift>
1440 return SelectT2AddrModeImm7Offset(
Op,
N, OffImm, Shift);
1446 unsigned Opcode =
Op->getOpcode();
1450 AM = cast<LoadSDNode>(
Op)->getAddressingMode();
1453 AM = cast<StoreSDNode>(
Op)->getAddressingMode();
1456 AM = cast<MaskedLoadSDNode>(
Op)->getAddressingMode();
1459 AM = cast<MaskedStoreSDNode>(
Op)->getAddressingMode();
1470 ? CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N), MVT::i32)
1471 : CurDAG->getTargetConstant(-RHSC * (1 << Shift),
SDLoc(
N),
1478template <
int Min,
int Max>
1479bool ARMDAGToDAGISel::SelectImmediateInRange(
SDValue N,
SDValue &OffImm) {
1482 OffImm = CurDAG->getTargetConstant(Val,
SDLoc(
N), MVT::i32);
1488bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(
SDValue N,
1492 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N))
1496 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1497 int RHSC = (int)
RHS->getZExtValue();
1498 if (RHSC >= 0 && RHSC < 0x1000)
1500 else if (RHSC < 0 && RHSC >= -255)
1506 Base =
N.getOperand(0);
1507 OffReg =
N.getOperand(1);
1521 ShAmt = Sh->getZExtValue();
1522 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1533 unsigned PowerOfTwo = 0;
1535 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1537 replaceDAGValue(OffReg.
getOperand(1), NewMulConst);
1543 ShImm = CurDAG->getTargetConstant(ShAmt,
SDLoc(
N), MVT::i32);
1553 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i32);
1555 if (
N.getOpcode() !=
ISD::ADD || !CurDAG->isBaseWithConstantOffset(
N))
1563 if (RHSC > 1020 || RHSC % 4 != 0)
1566 Base =
N.getOperand(0);
1568 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1569 Base = CurDAG->getTargetFrameIndex(
1570 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1573 OffImm = CurDAG->getTargetConstant(RHSC/4,
SDLoc(
N), MVT::i32);
1584void ARMDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
1586 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {
MemOp});
1589bool ARMDAGToDAGISel::tryARMIndexedLoad(
SDNode *
N) {
1595 EVT LoadedVT =
LD->getMemoryVT();
1598 unsigned Opcode = 0;
1600 if (LoadedVT == MVT::i32 && isPre &&
1601 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1602 Opcode = ARM::LDR_PRE_IMM;
1604 }
else if (LoadedVT == MVT::i32 && !isPre &&
1605 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1606 Opcode = ARM::LDR_POST_IMM;
1608 }
else if (LoadedVT == MVT::i32 &&
1609 SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1610 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1613 }
else if (LoadedVT == MVT::i16 &&
1614 SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1617 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1618 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1619 }
else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1621 if (SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1623 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1627 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1629 Opcode = ARM::LDRB_PRE_IMM;
1630 }
else if (!isPre &&
1631 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1633 Opcode = ARM::LDRB_POST_IMM;
1634 }
else if (SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1636 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1642 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1646 CurDAG->getRegister(0, MVT::i32), Chain };
1647 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1649 transferMemOperands(
N, New);
1650 ReplaceNode(
N, New);
1656 CurDAG->getRegister(0, MVT::i32), Chain };
1657 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1659 transferMemOperands(
N, New);
1660 ReplaceNode(
N, New);
1668bool ARMDAGToDAGISel::tryT1IndexedLoad(
SDNode *
N) {
1670 EVT LoadedVT =
LD->getMemoryVT();
1676 auto *COffs = dyn_cast<ConstantSDNode>(
LD->getOffset());
1677 if (!COffs || COffs->getZExtValue() != 4)
1687 CurDAG->getRegister(0, MVT::i32), Chain };
1688 SDNode *
New = CurDAG->getMachineNode(ARM::tLDR_postidx,
SDLoc(
N), MVT::i32,
1689 MVT::i32, MVT::Other, Ops);
1690 transferMemOperands(
N, New);
1691 ReplaceNode(
N, New);
1695bool ARMDAGToDAGISel::tryT2IndexedLoad(
SDNode *
N) {
1701 EVT LoadedVT =
LD->getMemoryVT();
1705 unsigned Opcode = 0;
1707 if (SelectT2AddrModeImm8Offset(
N,
LD->getOffset(),
Offset)) {
1710 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1714 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1716 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1721 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1723 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1735 CurDAG->getRegister(0, MVT::i32), Chain };
1736 SDNode *
New = CurDAG->getMachineNode(Opcode,
SDLoc(
N), MVT::i32, MVT::i32,
1738 transferMemOperands(
N, New);
1739 ReplaceNode(
N, New);
1746bool ARMDAGToDAGISel::tryMVEIndexedLoad(
SDNode *
N) {
1748 unsigned Opcode = 0;
1749 bool isSExtLd, isPre;
1759 LoadedVT =
LD->getMemoryVT();
1763 Chain =
LD->getChain();
1766 Alignment =
LD->getAlign();
1770 PredReg = CurDAG->getRegister(0, MVT::i32);
1775 LoadedVT =
LD->getMemoryVT();
1779 Chain =
LD->getChain();
1782 Alignment =
LD->getAlign();
1786 PredReg =
LD->getMask();
1793 bool CanChangeType = Subtarget->
isLittle() && !isa<MaskedLoadSDNode>(
N);
1796 if (Alignment >=
Align(2) && LoadedVT == MVT::v4i16 &&
1797 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1)) {
1799 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1801 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1802 }
else if (LoadedVT == MVT::v8i8 &&
1803 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1805 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1807 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1808 }
else if (LoadedVT == MVT::v4i8 &&
1809 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1811 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1813 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1814 }
else if (Alignment >=
Align(4) &&
1815 (CanChangeType || LoadedVT == MVT::v4i32 ||
1816 LoadedVT == MVT::v4f32) &&
1817 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 2))
1818 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1819 else if (Alignment >=
Align(2) &&
1820 (CanChangeType || LoadedVT == MVT::v8i16 ||
1821 LoadedVT == MVT::v8f16) &&
1822 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1))
1823 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1824 else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
1825 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0))
1826 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1832 CurDAG->getTargetConstant(Pred,
SDLoc(
N), MVT::i32),
1834 CurDAG->getRegister(0, MVT::i32),
1837 N->getValueType(0), MVT::Other, Ops);
1838 transferMemOperands(
N, New);
1842 CurDAG->RemoveDeadNode(
N);
1850 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1851 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1852 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1853 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1854 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1861 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1862 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1863 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1864 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1865 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1871 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1873 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1874 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1875 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1876 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1882 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1884 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1885 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1886 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1887 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1895 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1896 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1897 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1898 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1899 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1900 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1901 V2, SubReg2, V3, SubReg3 };
1902 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1909 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1911 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1912 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1913 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1914 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1915 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1916 V2, SubReg2, V3, SubReg3 };
1917 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1924 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1926 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1927 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1928 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1929 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1930 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1931 V2, SubReg2, V3, SubReg3 };
1932 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1939 unsigned NumVecs,
bool is64BitVector) {
1940 unsigned NumRegs = NumVecs;
1941 if (!is64BitVector && NumVecs < 3)
1944 unsigned Alignment =
Align->getAsZExtVal();
1945 if (Alignment >= 32 && NumRegs == 4)
1947 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1949 else if (Alignment >= 8)
1954 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1960 default:
return false;
1961 case ARM::VLD1d8wb_fixed :
return true;
1962 case ARM::VLD1d16wb_fixed :
return true;
1963 case ARM::VLD1d64Qwb_fixed :
return true;
1964 case ARM::VLD1d32wb_fixed :
return true;
1965 case ARM::VLD1d64wb_fixed :
return true;
1966 case ARM::VLD1d8TPseudoWB_fixed :
return true;
1967 case ARM::VLD1d16TPseudoWB_fixed :
return true;
1968 case ARM::VLD1d32TPseudoWB_fixed :
return true;
1969 case ARM::VLD1d64TPseudoWB_fixed :
return true;
1970 case ARM::VLD1d8QPseudoWB_fixed :
return true;
1971 case ARM::VLD1d16QPseudoWB_fixed :
return true;
1972 case ARM::VLD1d32QPseudoWB_fixed :
return true;
1973 case ARM::VLD1d64QPseudoWB_fixed :
return true;
1974 case ARM::VLD1q8wb_fixed :
return true;
1975 case ARM::VLD1q16wb_fixed :
return true;
1976 case ARM::VLD1q32wb_fixed :
return true;
1977 case ARM::VLD1q64wb_fixed :
return true;
1978 case ARM::VLD1DUPd8wb_fixed :
return true;
1979 case ARM::VLD1DUPd16wb_fixed :
return true;
1980 case ARM::VLD1DUPd32wb_fixed :
return true;
1981 case ARM::VLD1DUPq8wb_fixed :
return true;
1982 case ARM::VLD1DUPq16wb_fixed :
return true;
1983 case ARM::VLD1DUPq32wb_fixed :
return true;
1984 case ARM::VLD2d8wb_fixed :
return true;
1985 case ARM::VLD2d16wb_fixed :
return true;
1986 case ARM::VLD2d32wb_fixed :
return true;
1987 case ARM::VLD2q8PseudoWB_fixed :
return true;
1988 case ARM::VLD2q16PseudoWB_fixed :
return true;
1989 case ARM::VLD2q32PseudoWB_fixed :
return true;
1990 case ARM::VLD2DUPd8wb_fixed :
return true;
1991 case ARM::VLD2DUPd16wb_fixed :
return true;
1992 case ARM::VLD2DUPd32wb_fixed :
return true;
1993 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return true;
1994 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return true;
1995 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return true;
2002 default:
return false;
2003 case ARM::VST1d8wb_fixed :
return true;
2004 case ARM::VST1d16wb_fixed :
return true;
2005 case ARM::VST1d32wb_fixed :
return true;
2006 case ARM::VST1d64wb_fixed :
return true;
2007 case ARM::VST1q8wb_fixed :
return true;
2008 case ARM::VST1q16wb_fixed :
return true;
2009 case ARM::VST1q32wb_fixed :
return true;
2010 case ARM::VST1q64wb_fixed :
return true;
2011 case ARM::VST1d8TPseudoWB_fixed :
return true;
2012 case ARM::VST1d16TPseudoWB_fixed :
return true;
2013 case ARM::VST1d32TPseudoWB_fixed :
return true;
2014 case ARM::VST1d64TPseudoWB_fixed :
return true;
2015 case ARM::VST1d8QPseudoWB_fixed :
return true;
2016 case ARM::VST1d16QPseudoWB_fixed :
return true;
2017 case ARM::VST1d32QPseudoWB_fixed :
return true;
2018 case ARM::VST1d64QPseudoWB_fixed :
return true;
2019 case ARM::VST2d8wb_fixed :
return true;
2020 case ARM::VST2d16wb_fixed :
return true;
2021 case ARM::VST2d32wb_fixed :
return true;
2022 case ARM::VST2q8PseudoWB_fixed :
return true;
2023 case ARM::VST2q16PseudoWB_fixed :
return true;
2024 case ARM::VST2q32PseudoWB_fixed :
return true;
2032 &&
"Incorrect fixed stride updating instruction.");
2035 case ARM::VLD1d8wb_fixed:
return ARM::VLD1d8wb_register;
2036 case ARM::VLD1d16wb_fixed:
return ARM::VLD1d16wb_register;
2037 case ARM::VLD1d32wb_fixed:
return ARM::VLD1d32wb_register;
2038 case ARM::VLD1d64wb_fixed:
return ARM::VLD1d64wb_register;
2039 case ARM::VLD1q8wb_fixed:
return ARM::VLD1q8wb_register;
2040 case ARM::VLD1q16wb_fixed:
return ARM::VLD1q16wb_register;
2041 case ARM::VLD1q32wb_fixed:
return ARM::VLD1q32wb_register;
2042 case ARM::VLD1q64wb_fixed:
return ARM::VLD1q64wb_register;
2043 case ARM::VLD1d64Twb_fixed:
return ARM::VLD1d64Twb_register;
2044 case ARM::VLD1d64Qwb_fixed:
return ARM::VLD1d64Qwb_register;
2045 case ARM::VLD1d8TPseudoWB_fixed:
return ARM::VLD1d8TPseudoWB_register;
2046 case ARM::VLD1d16TPseudoWB_fixed:
return ARM::VLD1d16TPseudoWB_register;
2047 case ARM::VLD1d32TPseudoWB_fixed:
return ARM::VLD1d32TPseudoWB_register;
2048 case ARM::VLD1d64TPseudoWB_fixed:
return ARM::VLD1d64TPseudoWB_register;
2049 case ARM::VLD1d8QPseudoWB_fixed:
return ARM::VLD1d8QPseudoWB_register;
2050 case ARM::VLD1d16QPseudoWB_fixed:
return ARM::VLD1d16QPseudoWB_register;
2051 case ARM::VLD1d32QPseudoWB_fixed:
return ARM::VLD1d32QPseudoWB_register;
2052 case ARM::VLD1d64QPseudoWB_fixed:
return ARM::VLD1d64QPseudoWB_register;
2053 case ARM::VLD1DUPd8wb_fixed :
return ARM::VLD1DUPd8wb_register;
2054 case ARM::VLD1DUPd16wb_fixed :
return ARM::VLD1DUPd16wb_register;
2055 case ARM::VLD1DUPd32wb_fixed :
return ARM::VLD1DUPd32wb_register;
2056 case ARM::VLD1DUPq8wb_fixed :
return ARM::VLD1DUPq8wb_register;
2057 case ARM::VLD1DUPq16wb_fixed :
return ARM::VLD1DUPq16wb_register;
2058 case ARM::VLD1DUPq32wb_fixed :
return ARM::VLD1DUPq32wb_register;
2059 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return ARM::VLD2DUPq8OddPseudoWB_register;
2060 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return ARM::VLD2DUPq16OddPseudoWB_register;
2061 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return ARM::VLD2DUPq32OddPseudoWB_register;
2063 case ARM::VST1d8wb_fixed:
return ARM::VST1d8wb_register;
2064 case ARM::VST1d16wb_fixed:
return ARM::VST1d16wb_register;
2065 case ARM::VST1d32wb_fixed:
return ARM::VST1d32wb_register;
2066 case ARM::VST1d64wb_fixed:
return ARM::VST1d64wb_register;
2067 case ARM::VST1q8wb_fixed:
return ARM::VST1q8wb_register;
2068 case ARM::VST1q16wb_fixed:
return ARM::VST1q16wb_register;
2069 case ARM::VST1q32wb_fixed:
return ARM::VST1q32wb_register;
2070 case ARM::VST1q64wb_fixed:
return ARM::VST1q64wb_register;
2071 case ARM::VST1d8TPseudoWB_fixed:
return ARM::VST1d8TPseudoWB_register;
2072 case ARM::VST1d16TPseudoWB_fixed:
return ARM::VST1d16TPseudoWB_register;
2073 case ARM::VST1d32TPseudoWB_fixed:
return ARM::VST1d32TPseudoWB_register;
2074 case ARM::VST1d64TPseudoWB_fixed:
return ARM::VST1d64TPseudoWB_register;
2075 case ARM::VST1d8QPseudoWB_fixed:
return ARM::VST1d8QPseudoWB_register;
2076 case ARM::VST1d16QPseudoWB_fixed:
return ARM::VST1d16QPseudoWB_register;
2077 case ARM::VST1d32QPseudoWB_fixed:
return ARM::VST1d32QPseudoWB_register;
2078 case ARM::VST1d64QPseudoWB_fixed:
return ARM::VST1d64QPseudoWB_register;
2080 case ARM::VLD2d8wb_fixed:
return ARM::VLD2d8wb_register;
2081 case ARM::VLD2d16wb_fixed:
return ARM::VLD2d16wb_register;
2082 case ARM::VLD2d32wb_fixed:
return ARM::VLD2d32wb_register;
2083 case ARM::VLD2q8PseudoWB_fixed:
return ARM::VLD2q8PseudoWB_register;
2084 case ARM::VLD2q16PseudoWB_fixed:
return ARM::VLD2q16PseudoWB_register;
2085 case ARM::VLD2q32PseudoWB_fixed:
return ARM::VLD2q32PseudoWB_register;
2087 case ARM::VST2d8wb_fixed:
return ARM::VST2d8wb_register;
2088 case ARM::VST2d16wb_fixed:
return ARM::VST2d16wb_register;
2089 case ARM::VST2d32wb_fixed:
return ARM::VST2d32wb_register;
2090 case ARM::VST2q8PseudoWB_fixed:
return ARM::VST2q8PseudoWB_register;
2091 case ARM::VST2q16PseudoWB_fixed:
return ARM::VST2q16PseudoWB_register;
2092 case ARM::VST2q32PseudoWB_fixed:
return ARM::VST2q32PseudoWB_register;
2094 case ARM::VLD2DUPd8wb_fixed:
return ARM::VLD2DUPd8wb_register;
2095 case ARM::VLD2DUPd16wb_fixed:
return ARM::VLD2DUPd16wb_register;
2096 case ARM::VLD2DUPd32wb_fixed:
return ARM::VLD2DUPd32wb_register;
2105 auto C = dyn_cast<ConstantSDNode>(Inc);
2109void ARMDAGToDAGISel::SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2113 assert(Subtarget->hasNEON());
2114 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLD NumVecs out-of-range");
2118 bool IsIntrinsic = !isUpdating;
2120 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2121 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2125 EVT VT =
N->getValueType(0);
2127 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2155 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2160 std::vector<EVT> ResTys;
2161 ResTys.push_back(ResTy);
2163 ResTys.push_back(MVT::i32);
2164 ResTys.push_back(MVT::Other);
2167 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2172 if (is64BitVector || NumVecs <= 2) {
2173 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2178 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2194 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2204 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2205 const SDValue OpsA[] = { MemAddr,
Align, Reg0, ImplDef, Pred, Reg0, Chain };
2207 ResTy, AddrTy, MVT::Other, OpsA);
2214 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2216 "only constant post-increment update allowed for VLD3/4");
2224 VLd = CurDAG->getMachineNode(QOpcodes1[
OpcodeIndex], dl, ResTys, Ops);
2229 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {
MemOp});
2232 ReplaceNode(
N, VLd);
2238 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2239 ARM::qsub_3 == ARM::qsub_0 + 3,
2240 "Unexpected subreg numbering");
2241 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2242 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2244 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2248 CurDAG->RemoveDeadNode(
N);
2251void ARMDAGToDAGISel::SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2255 assert(Subtarget->hasNEON());
2256 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VST NumVecs out-of-range");
2260 bool IsIntrinsic = !isUpdating;
2262 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2263 unsigned Vec0Idx = 3;
2264 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2270 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2272 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2296 std::vector<EVT> ResTys;
2298 ResTys.push_back(MVT::i32);
2299 ResTys.push_back(MVT::Other);
2302 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2306 if (is64BitVector || NumVecs <= 2) {
2309 SrcReg =
N->getOperand(Vec0Idx);
2310 }
else if (is64BitVector) {
2312 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2313 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2315 SrcReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2321 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2322 :
N->getOperand(Vec0Idx + 3);
2323 SrcReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2327 SDValue Q0 =
N->getOperand(Vec0Idx);
2328 SDValue Q1 =
N->getOperand(Vec0Idx + 1);
2329 SrcReg =
SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2332 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2337 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2355 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2358 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {
MemOp});
2360 ReplaceNode(
N, VSt);
2368 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2369 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2372 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2373 :
N->getOperand(Vec0Idx + 3);
2374 SDValue RegSeq =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2378 const SDValue OpsA[] = { MemAddr,
Align, Reg0, RegSeq, Pred, Reg0, Chain };
2382 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {
MemOp});
2389 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2391 "only constant post-increment update allowed for VST3/4");
2401 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {
MemOp});
2402 ReplaceNode(
N, VStB);
2405void ARMDAGToDAGISel::SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
2409 assert(Subtarget->hasNEON());
2410 assert(NumVecs >=2 && NumVecs <= 4 &&
"VLDSTLane NumVecs out-of-range");
2414 bool IsIntrinsic = !isUpdating;
2416 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2417 unsigned Vec0Idx = 3;
2418 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2424 unsigned Lane =
N->getConstantOperandVal(Vec0Idx + NumVecs);
2425 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2428 unsigned Alignment = 0;
2430 Alignment =
Align->getAsZExtVal();
2432 if (Alignment > NumBytes)
2433 Alignment = NumBytes;
2434 if (Alignment < 8 && Alignment < NumBytes)
2437 Alignment = (Alignment & -Alignment);
2441 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2461 std::vector<EVT> ResTys;
2463 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2467 MVT::i64, ResTyElts));
2470 ResTys.push_back(MVT::i32);
2471 ResTys.push_back(MVT::Other);
2474 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2480 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2483 Ops.
push_back(IsImmUpdate ? Reg0 : Inc);
2487 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2488 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2491 SuperReg =
SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2493 SuperReg =
SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2497 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2498 :
N->getOperand(Vec0Idx + 3);
2500 SuperReg =
SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2502 SuperReg =
SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2510 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2512 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2513 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {
MemOp});
2515 ReplaceNode(
N, VLdLn);
2521 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2522 ARM::qsub_3 == ARM::qsub_0 + 3,
2523 "Unexpected subreg numbering");
2524 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2525 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2527 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2531 CurDAG->RemoveDeadNode(
N);
2534template <
typename SDValueVector>
2535void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2537 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2538 Ops.push_back(PredicateMask);
2539 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2542template <
typename SDValueVector>
2543void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2546 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::Then, Loc, MVT::i32));
2547 Ops.push_back(PredicateMask);
2548 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2549 Ops.push_back(Inactive);
2552template <
typename SDValueVector>
2553void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc) {
2554 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2555 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2556 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2559template <
typename SDValueVector>
2560void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2562 Ops.push_back(CurDAG->getTargetConstant(
ARMVCC::None, Loc, MVT::i32));
2563 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2564 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2566 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2575 switch (
N->getValueType(1).getVectorElementType().getSizeInBits()) {
2577 Opcode = Opcodes[0];
2580 Opcode = Opcodes[1];
2588 int32_t ImmValue =
N->getConstantOperandVal(3);
2589 Ops.
push_back(getI32Imm(ImmValue, Loc));
2592 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2594 AddEmptyMVEPredicateToOps(Ops, Loc);
2607 transferMemOperands(
N, New);
2608 CurDAG->RemoveDeadNode(
N);
2613 bool HasSaturationOperand) {
2623 int32_t ImmValue =
N->getConstantOperandVal(3);
2624 Ops.
push_back(getI32Imm(ImmValue, Loc));
2630 if (HasSaturationOperand) {
2631 int32_t SatOp =
N->getConstantOperandVal(4);
2632 int SatBit = (SatOp == 64 ? 0 : 1);
2639 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
2641 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2644void ARMDAGToDAGISel::SelectMVE_VADCSBC(
SDNode *
N,
uint16_t OpcodeWithCarry,
2646 bool Add,
bool Predicated) {
2651 unsigned FirstInputOp = Predicated ? 2 : 1;
2655 Ops.
push_back(
N->getOperand(FirstInputOp + 1));
2656 SDValue CarryIn =
N->getOperand(FirstInputOp + 2);
2657 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2660 if (CarryInConstant &&
2661 (CarryInConstant->
getZExtValue() & CarryMask) == CarryExpected) {
2662 Opcode = OpcodeWithNoCarry;
2665 Opcode = OpcodeWithCarry;
2669 AddMVEPredicateToOps(Ops, Loc,
2670 N->getOperand(FirstInputOp + 3),
2671 N->getOperand(FirstInputOp - 1));
2673 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2675 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2678void ARMDAGToDAGISel::SelectMVE_VSHLC(
SDNode *
N,
bool Predicated) {
2686 int32_t ImmValue =
N->getConstantOperandVal(3);
2687 Ops.
push_back(getI32Imm(ImmValue, Loc));
2690 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2692 AddEmptyMVEPredicateToOps(Ops, Loc);
2694 CurDAG->SelectNodeTo(
N, ARM::MVE_VSHLC,
N->getVTList(),
ArrayRef(Ops));
2698 assert(isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant");
2705void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2708 size_t Stride,
size_t TySize) {
2709 assert(TySize < Stride &&
"Invalid TySize");
2715 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2717 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2720 auto OpIsZero = [
N](
size_t OpNo) {
2726 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2728 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2730 Opcodes += 4 * Stride;
2732 Opcodes += 2 * Stride;
2749 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(8));
2751 AddEmptyMVEPredicateToOps(Ops, Loc);
2753 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2756void ARMDAGToDAGISel::SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2759 EVT VecTy =
N->getOperand(6).getValueType();
2772 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2775void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
2779 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2781 "bad vector element size");
2782 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 1, 0);
2785void ARMDAGToDAGISel::SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
2787 bool HasWriteback) {
2788 EVT VT =
N->getValueType(0);
2794 OurOpcodes = Opcodes[0];
2797 OurOpcodes = Opcodes[1];
2800 OurOpcodes = Opcodes[2];
2808 unsigned PtrOperand = HasWriteback ? 1 : 2;
2811 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2814 for (
unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2815 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2817 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2824 ResultTys = {DataTy, MVT::i32, MVT::Other};
2825 SDValue Ops[] = {Data,
N->getOperand(PtrOperand), Chain};
2827 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2831 for (i = 0; i < NumVecs; i++)
2833 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2838 CurDAG->RemoveDeadNode(
N);
2841void ARMDAGToDAGISel::SelectMVE_VxDUP(
SDNode *
N,
const uint16_t *Opcodes,
2842 bool Wrapping,
bool Predicated) {
2843 EVT VT =
N->getValueType(0);
2849 Opcode = Opcodes[0];
2852 Opcode = Opcodes[1];
2855 Opcode = Opcodes[2];
2866 Inactive =
N->getOperand(OpIdx++);
2872 SDValue ImmOp =
N->getOperand(OpIdx++);
2874 Ops.
push_back(getI32Imm(ImmValue, Loc));
2877 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(OpIdx), Inactive);
2879 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2881 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2885 size_t NumExtraOps,
bool HasAccum) {
2886 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2893 SDValue ImmCorpoc =
N->getOperand(OpIdx++);
2895 Ops.
push_back(getI32Imm(ImmCoprocVal, Loc));
2900 SDValue AccLo =
N->getOperand(OpIdx++);
2901 SDValue AccHi =
N->getOperand(OpIdx++);
2908 for (
size_t I = 0;
I < NumExtraOps;
I++)
2919 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2925 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops);
2932 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2936 for (
size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2937 if (
SDValue(
N, ResIdx).use_empty())
2939 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2940 MVT::i32, ResultPair);
2944 CurDAG->RemoveDeadNode(
N);
2947void ARMDAGToDAGISel::SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
2948 bool isUpdating,
unsigned NumVecs,
2952 assert(Subtarget->hasNEON());
2953 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLDDup NumVecs out-of-range");
2957 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2958 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2962 EVT VT =
N->getValueType(0);
2965 unsigned Alignment = 0;
2967 Alignment =
Align->getAsZExtVal();
2969 if (Alignment > NumBytes)
2970 Alignment = NumBytes;
2971 if (Alignment < 8 && Alignment < NumBytes)
2974 Alignment = (Alignment & -Alignment);
2978 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
3000 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
3005 std::vector<EVT> ResTys;
3006 ResTys.push_back(ResTy);
3008 ResTys.push_back(MVT::i32);
3009 ResTys.push_back(MVT::Other);
3012 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3017 unsigned Opc = is64BitVector ? DOpcodes[
OpcodeIndex]
3033 if (is64BitVector || NumVecs == 1) {
3037 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3038 const SDValue OpsA[] = {MemAddr,
Align, ImplDef, Pred, Reg0, Chain};
3049 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3053 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {
MemOp});
3060 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7,
"Unexpected subreg numbering");
3061 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3062 for (
unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3064 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3070 CurDAG->RemoveDeadNode(
N);
3073bool ARMDAGToDAGISel::tryInsertVectorElt(
SDNode *
N) {
3074 if (!Subtarget->hasMVEIntegerOps())
3088 (VT != MVT::v8f16 && VT != MVT::v8i16) || (Ins2.
getValueType() != VT))
3093 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3120 ExtractLane1 == ExtractLane2 + 1) {
3121 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3122 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val1.
getOperand(0));
3123 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3124 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.
getOperand(0),
3126 ReplaceUses(Ins1, NewIns);
3132 if (VT == MVT::v8i16 && Subtarget->hasFullFP16()) {
3133 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3134 ARM::ssub_0 + ExtractLane1 / 2, dl, MVT::f32, Val1.
getOperand(0));
3135 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3136 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val2.
getOperand(0));
3137 if (ExtractLane1 % 2 != 0)
3138 Inp1 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0);
3139 if (ExtractLane2 % 2 != 0)
3140 Inp2 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0);
3141 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1);
3143 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3145 ReplaceUses(Ins1, NewIns);
3152 if (VT == MVT::v8f16 && Subtarget->hasFullFP16()) {
3153 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1);
3155 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3157 ReplaceUses(Ins1, NewIns);
3164bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(
SDNode *
N,
3167 bool FixedToFloat) {
3168 auto Type =
N->getValueType(0);
3170 if (ScalarBits > 32)
3176 if (ScalarBits == 16 && !FMulFlags.
hasNoInfs() && IsUnsigned)
3201 if (!isa<ConstantSDNode>(ImmNode.
getOperand(0)))
3207 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3208 APInt(ScalarBits, Imm));
3225 if (!ImmAPF.getExactInverse(&ToConvert))
3228 APSInt Converted(64,
false);
3232 if (!IsExact || !Converted.isPowerOf2())
3235 unsigned FracBits = Converted.logBase2();
3236 if (FracBits > ScalarBits)
3240 VecVal, CurDAG->getConstant(FracBits,
SDLoc(
N), MVT::i32)};
3241 AddEmptyMVEPredicateToOps(Ops,
SDLoc(
N),
Type);
3243 unsigned int Opcode;
3244 switch (ScalarBits) {
3247 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3249 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3253 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3255 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3262 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
SDLoc(
N),
Type, Ops));
3266bool ARMDAGToDAGISel::tryFP_TO_INT(
SDNode *
N,
SDLoc dl) {
3268 if (!Subtarget->hasMVEFloatOps())
3271 if (!
Type.isVector())
3282 if (
Node->getOperand(0) !=
Node->getOperand(1))
3287 if (ScalarBits == 16 && !
Flags.hasNoInfs() && IsUnsigned)
3291 switch (ScalarBits) {
3293 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3296 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3300 CurDAG->getConstant(1, dl, MVT::i32)};
3301 AddEmptyMVEPredicateToOps(Ops, dl,
Type);
3303 ReplaceNode(
N, CurDAG->getMachineNode(Opcode, dl,
Type, Ops));
3310 return transformFixedFloatingPointConversion(
N,
Node, IsUnsigned,
false);
3313bool ARMDAGToDAGISel::tryFMULFixed(
SDNode *
N,
SDLoc dl) {
3315 if (!Subtarget->hasMVEFloatOps())
3317 auto Type =
N->getValueType(0);
3318 if (!
Type.isVector())
3321 auto LHS =
N->getOperand(0);
3325 return transformFixedFloatingPointConversion(
3329bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned) {
3330 if (!Subtarget->hasV6T2Ops())
3334 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3335 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3339 unsigned And_imm = 0;
3344 if (And_imm & (And_imm + 1))
3347 unsigned Srl_imm = 0;
3350 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3355 And_imm &= -1U >> Srl_imm;
3359 unsigned LSB = Srl_imm;
3361 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3363 if ((LSB + Width + 1) ==
N->getValueType(0).getSizeInBits()) {
3365 if (Subtarget->isThumb()) {
3366 Opc =
isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3367 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3368 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3369 getAL(CurDAG, dl), Reg0, Reg0 };
3370 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3379 SDValue Ops[] = {
N->getOperand(0).getOperand(0), ShOpc,
3380 getAL(CurDAG, dl), Reg0, Reg0 };
3381 CurDAG->SelectNodeTo(
N, ARM::MOVsi, MVT::i32, Ops);
3385 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3386 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3387 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3388 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3389 getAL(CurDAG, dl), Reg0 };
3390 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3398 unsigned Shl_imm = 0;
3400 assert(Shl_imm > 0 && Shl_imm < 32 &&
"bad amount in shift node!");
3401 unsigned Srl_imm = 0;
3403 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3405 unsigned Width = 32 - Srl_imm - 1;
3406 int LSB = Srl_imm - Shl_imm;
3409 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3410 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3411 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3412 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3413 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3414 getAL(CurDAG, dl), Reg0 };
3415 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3423 unsigned Srl_imm = 0;
3427 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3430 unsigned Width = MSB - LSB;
3431 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3432 assert(Srl_imm + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3433 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3434 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
3435 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3436 getAL(CurDAG, dl), Reg0 };
3437 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3443 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3449 if (LSB + Width > 32)
3452 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3453 assert(LSB + Width <= 32 &&
"Shouldn't create an invalid ubfx");
3454 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3455 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3456 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
3457 getAL(CurDAG, dl), Reg0 };
3458 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3475bool ARMDAGToDAGISel::tryABSOp(
SDNode *
N){
3476 SDValue SUBSrc0 =
N->getOperand(0);
3477 SDValue SUBSrc1 =
N->getOperand(1);
3478 EVT VT =
N->getValueType(0);
3494 if (XORSrc1 == SUBSrc1 && XORSrc0 == SRASrc0 && XType.
isInteger() &&
3496 unsigned Opcode = Subtarget->
isThumb2() ? ARM::t2ABS : ARM::ABS;
3497 CurDAG->SelectNodeTo(
N, Opcode, VT, XORSrc0);
3505void ARMDAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3507 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3508 if (MemTy == MVT::i8)
3509 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3510 else if (MemTy == MVT::i16)
3511 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3512 else if (MemTy == MVT::i32)
3513 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32;
3517 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3519 SDNode *CmpSwap = CurDAG->getMachineNode(
3521 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
3524 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3528 CurDAG->RemoveDeadNode(
N);
3531static std::optional<std::pair<unsigned, unsigned>>
3533 unsigned FirstOne =
A.getBitWidth() -
A.countl_zero() - 1;
3534 unsigned LastOne =
A.countr_zero();
3535 if (
A.popcount() != (FirstOne - LastOne + 1))
3536 return std::nullopt;
3537 return std::make_pair(FirstOne, LastOne);
3540void ARMDAGToDAGISel::SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI) {
3542 SwitchEQNEToPLMI =
false;
3544 if (!Subtarget->isThumb())
3551 if (!
And->hasOneUse())
3558 auto C = dyn_cast<ConstantSDNode>(
And.getOperand(1));
3572 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3573 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3574 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3575 CurDAG->getRegister(0, MVT::i32) };
3576 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3578 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
3579 CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3580 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3581 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3585 if (
Range->second == 0) {
3587 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3588 ReplaceNode(
And.getNode(), NewN);
3589 }
else if (
Range->first == 31) {
3591 NewN = EmitShift(ARM::tLSRri,
X,
Range->second);
3592 ReplaceNode(
And.getNode(), NewN);
3596 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3597 ReplaceNode(
And.getNode(), NewN);
3599 SwitchEQNEToPLMI =
true;
3600 }
else if (!Subtarget->hasV6T2Ops()) {
3603 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3604 NewN = EmitShift(ARM::tLSRri,
SDValue(NewN, 0),
3606 ReplaceNode(
And.getNode(), NewN);
3611 unsigned Opc128[3]) {
3613 "Unexpected vector shuffle length");
3626void ARMDAGToDAGISel::Select(
SDNode *
N) {
3629 if (
N->isMachineOpcode()) {
3634 switch (
N->getOpcode()) {
3656 cast<RegisterSDNode>(
Ptr.getOperand(1))->getReg() == ARM::SP &&
3657 Ptr.getOperand(0) ==
ST->getChain()) {
3659 CurDAG->getRegister(ARM::SP, MVT::i32),
3660 CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
3662 CurDAG->getRegister(0, MVT::i32),
3665 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
3667 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3668 ReplaceNode(
N, ResNode);
3675 if (tryWriteRegister(
N))
3679 if (tryReadRegister(
N))
3684 if (tryInlineAsm(
N))
3694 unsigned Val =
N->getAsZExtVal();
3697 !Subtarget->genExecuteOnly()) {
3698 SDValue CPIdx = CurDAG->getTargetConstantPool(
3700 TLI->getPointerTy(CurDAG->getDataLayout()));
3703 if (Subtarget->isThumb()) {
3707 CurDAG->getRegister(0, MVT::i32),
3708 CurDAG->getEntryNode()
3710 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
3715 CurDAG->getTargetConstant(0, dl, MVT::i32),
3717 CurDAG->getRegister(0, MVT::i32),
3718 CurDAG->getEntryNode()
3720 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
3731 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3733 ReplaceNode(
N, ResNode);
3742 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
3743 SDValue TFI = CurDAG->getTargetFrameIndex(
3744 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3751 CurDAG->SelectNodeTo(
N, ARM::tADDframe, MVT::i32, TFI,
3752 CurDAG->getTargetConstant(0, dl, MVT::i32));
3755 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3756 ARM::t2ADDri : ARM::ADDri);
3757 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3758 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3759 CurDAG->getRegister(0, MVT::i32) };
3760 CurDAG->SelectNodeTo(
N, Opc, MVT::i32, Ops);
3765 if (tryInsertVectorElt(
N))
3770 if (tryV6T2BitfieldExtractOp(
N,
false))
3775 if (tryV6T2BitfieldExtractOp(
N,
true))
3782 if (tryFP_TO_INT(
N, dl))
3786 if (tryFMULFixed(
N, dl))
3793 unsigned RHSV =
C->getZExtValue();
3796 unsigned ShImm =
Log2_32(RHSV-1);
3801 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3802 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3803 if (Subtarget->isThumb()) {
3804 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3805 CurDAG->SelectNodeTo(
N, ARM::t2ADDrs, MVT::i32, Ops);
3810 CurDAG->SelectNodeTo(
N, ARM::ADDrsi, MVT::i32, Ops);
3815 unsigned ShImm =
Log2_32(RHSV+1);
3820 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3821 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3822 if (Subtarget->isThumb()) {
3823 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3824 CurDAG->SelectNodeTo(
N, ARM::t2RSBrs, MVT::i32, Ops);
3829 CurDAG->SelectNodeTo(
N, ARM::RSBrsi, MVT::i32, Ops);
3837 if (tryV6T2BitfieldExtractOp(
N,
false))
3843 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3844 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3851 bool PreferImmediateEncoding =
3852 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3853 if (!PreferImmediateEncoding &&
3859 CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
3863 CurDAG->RepositionNode(
N->getIterator(), NewImm.
getNode());
3865 if (!Subtarget->hasThumb2()) {
3866 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3867 N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3868 CurDAG->getRegister(0, MVT::i32)};
3869 ReplaceNode(
N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
3872 SDValue Ops[] = {
N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3873 CurDAG->getRegister(0, MVT::i32),
3874 CurDAG->getRegister(0, MVT::i32)};
3876 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
3887 EVT VT =
N->getValueType(0);
3890 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3892 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3895 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
3896 N1C = dyn_cast<ConstantSDNode>(N1);
3904 unsigned N1CVal = N1C->getZExtValue();
3906 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3907 (N1CVal & 0xffffU) == 0xffffU &&
3908 (N2CVal & 0xffffU) == 0x0U) {
3909 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3912 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
3913 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3921 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3922 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
3923 N->getOperand(2),
N->getOperand(3),
3925 CurDAG->getRegister(0, MVT::i32) };
3926 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
3930 if (Subtarget->isThumb()) {
3931 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3932 N->getOperand(3),
getAL(CurDAG, dl),
3933 CurDAG->getRegister(0, MVT::i32)};
3935 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
3938 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3939 N->getOperand(3),
getAL(CurDAG, dl),
3940 CurDAG->getRegister(0, MVT::i32),
3941 CurDAG->getRegister(0, MVT::i32) };
3942 ReplaceNode(
N, CurDAG->getMachineNode(
3943 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3944 MVT::i32, MVT::i32, Ops));
3949 if (Subtarget->isThumb()) {
3950 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3951 N->getOperand(3),
getAL(CurDAG, dl),
3952 CurDAG->getRegister(0, MVT::i32)};
3954 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
3957 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3958 N->getOperand(3),
getAL(CurDAG, dl),
3959 CurDAG->getRegister(0, MVT::i32),
3960 CurDAG->getRegister(0, MVT::i32) };
3961 ReplaceNode(
N, CurDAG->getMachineNode(
3962 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3963 MVT::i32, MVT::i32, Ops));
3968 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3977 if (Subtarget->isThumb())
3978 assert(Subtarget->hasThumb2() &&
3979 "This pattern should not be generated for Thumb");
3981 SDValue SmulLoHi =
N->getOperand(1);
3986 N->getOperand(1) != SmulLoHi.
getValue(1) ||
3990 unsigned Opc = Subtarget->
isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3992 N->getOperand(0),
getAL(CurDAG, dl),
3993 CurDAG->getRegister(0, MVT::i32) };
3994 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
3998 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4000 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
4001 if (tryT2IndexedLoad(
N))
4003 }
else if (Subtarget->isThumb()) {
4004 if (tryT1IndexedLoad(
N))
4006 }
else if (tryARMIndexedLoad(
N))
4012 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4017 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopSetup, dl, MVT::i32,
4019 ReplaceUses(
N, New);
4020 CurDAG->RemoveDeadNode(
N);
4024 SDNode *
New = CurDAG->getMachineNode(ARM::t2WhileLoopStart, dl, MVT::Other,
4025 N->getOperand(1),
N->getOperand(2),
4027 ReplaceUses(
N, New);
4028 CurDAG->RemoveDeadNode(
N);
4032 SDValue Ops[] = {
N->getOperand(1),
4035 unsigned Opc = ARM::t2LoopEnd;
4036 SDNode *
New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
4037 ReplaceUses(
N, New);
4038 CurDAG->RemoveDeadNode(
N);
4045 const SDValue &Chain =
N->getOperand(0);
4047 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4048 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4054 RegOffset = CurDAG->getRegister(0, MVT::i32);
4056 SDValue Ops[] = {
Base, RegOffset, ImmOffset, Chain};
4057 SDNode *
New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4058 {MVT::Untyped, MVT::Other}, Ops);
4059 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4061 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4063 transferMemOperands(
N, New);
4067 CurDAG->RemoveDeadNode(
N);
4074 const SDValue &Chain =
N->getOperand(0);
4076 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4077 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4083 RegOffset = CurDAG->getRegister(0, MVT::i32);
4088 SDNode *
New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops);
4089 transferMemOperands(
N, New);
4091 CurDAG->RemoveDeadNode(
N);
4095 SDValue Ops[] = {
N->getOperand(1),
4099 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4100 CurDAG->getVTList(MVT::i32, MVT::Other), Ops);
4101 ReplaceUses(
N, Dec);
4102 CurDAG->RemoveDeadNode(
N);
4118 unsigned Opc = Subtarget->isThumb() ?
4119 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4137 if (
ID == Intrinsic::loop_decrement_reg) {
4139 SDValue Size = CurDAG->getTargetConstant(
Int.getConstantOperandVal(3),
4144 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4145 CurDAG->getVTList(MVT::i32, MVT::Other),
4147 ReplaceUses(
Int.getNode(), LoopDec);
4151 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
4153 ReplaceUses(
N, LoopEnd);
4154 CurDAG->RemoveDeadNode(
N);
4155 CurDAG->RemoveDeadNode(InGlue.
getNode());
4156 CurDAG->RemoveDeadNode(
Int.getNode());
4161 bool SwitchEQNEToPLMI;
4162 SelectCMPZ(InGlue.
getNode(), SwitchEQNEToPLMI);
4163 InGlue =
N->getOperand(4);
4165 if (SwitchEQNEToPLMI) {
4178 SDValue Tmp2 = CurDAG->getTargetConstant(
CC, dl, MVT::i32);
4179 SDValue Ops[] = { N1, Tmp2, N3, Chain, InGlue };
4180 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
4183 if (
N->getNumValues() == 2) {
4185 ReplaceUses(
SDValue(
N, 1), InGlue);
4189 CurDAG->RemoveDeadNode(
N);
4199 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1).getNode());
4200 if (
C &&
C->getSExtValue() < 0 && Subtarget->isThumb()) {
4201 int64_t Addend = -
C->getSExtValue();
4208 if (Addend < 1<<8) {
4210 SDValue Ops[] = {
X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4211 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
4212 CurDAG->getRegister(0, MVT::i32) };
4213 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
4215 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4216 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
X,
4217 CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4218 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
4219 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
4224 CurDAG->MorphNodeTo(
N,
ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
4235 bool SwitchEQNEToPLMI;
4236 SelectCMPZ(InGlue.
getNode(), SwitchEQNEToPLMI);
4238 if (SwitchEQNEToPLMI) {
4251 SDValue NewARMcc = CurDAG->getConstant((
unsigned)
CC, dl, MVT::i32);
4252 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), NewARMcc,
4253 N->getOperand(3),
N->getOperand(4)};
4262 EVT VT =
N->getValueType(0);
4264 unsigned Opc64[] = {ARM::VZIPd8, ARM::VZIPd16, ARM::VTRNd32};
4265 unsigned Opc128[] = {ARM::VZIPq8, ARM::VZIPq16, ARM::VZIPq32};
4268 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4269 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4270 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4274 EVT VT =
N->getValueType(0);
4276 unsigned Opc64[] = {ARM::VUZPd8, ARM::VUZPd16, ARM::VTRNd32};
4277 unsigned Opc128[] = {ARM::VUZPq8, ARM::VUZPq16, ARM::VUZPq32};
4280 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4281 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4282 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4286 EVT VT =
N->getValueType(0);
4287 unsigned Opc64[] = {ARM::VTRNd8, ARM::VTRNd16, ARM::VTRNd32};
4288 unsigned Opc128[] = {ARM::VTRNq8, ARM::VTRNq16, ARM::VTRNq32};
4291 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4292 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4293 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4297 EVT VecVT =
N->getValueType(0);
4300 if (EltVT == MVT::f64) {
4301 assert(NumElts == 2 &&
"unexpected type for BUILD_VECTOR");
4303 N, createDRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4306 assert(EltVT == MVT::f32 &&
"unexpected type for BUILD_VECTOR");
4309 N, createSRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4312 assert(NumElts == 4 &&
"unexpected type for BUILD_VECTOR");
4314 createQuadSRegsNode(VecVT,
N->getOperand(0),
N->getOperand(1),
4315 N->getOperand(2),
N->getOperand(3)));
4320 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4322 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4324 SelectVLDDup(
N,
false,
false, 1, DOpcodes, QOpcodes);
4329 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4331 SelectVLDDup(
N,
false,
false, 2, Opcodes);
4336 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4337 ARM::VLD3DUPd16Pseudo,
4338 ARM::VLD3DUPd32Pseudo };
4339 SelectVLDDup(
N,
false,
false, 3, Opcodes);
4344 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4345 ARM::VLD4DUPd16Pseudo,
4346 ARM::VLD4DUPd32Pseudo };
4347 SelectVLDDup(
N,
false,
false, 4, Opcodes);
4352 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4353 ARM::VLD1DUPd16wb_fixed,
4354 ARM::VLD1DUPd32wb_fixed };
4355 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4356 ARM::VLD1DUPq16wb_fixed,
4357 ARM::VLD1DUPq32wb_fixed };
4358 SelectVLDDup(
N,
false,
true, 1, DOpcodes, QOpcodes);
4363 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4364 ARM::VLD2DUPd16wb_fixed,
4365 ARM::VLD2DUPd32wb_fixed,
4366 ARM::VLD1q64wb_fixed };
4367 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4368 ARM::VLD2DUPq16EvenPseudo,
4369 ARM::VLD2DUPq32EvenPseudo };
4370 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4371 ARM::VLD2DUPq16OddPseudoWB_fixed,
4372 ARM::VLD2DUPq32OddPseudoWB_fixed };
4373 SelectVLDDup(
N,
false,
true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4378 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4379 ARM::VLD3DUPd16Pseudo_UPD,
4380 ARM::VLD3DUPd32Pseudo_UPD,
4381 ARM::VLD1d64TPseudoWB_fixed };
4382 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4383 ARM::VLD3DUPq16EvenPseudo,
4384 ARM::VLD3DUPq32EvenPseudo };
4385 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4386 ARM::VLD3DUPq16OddPseudo_UPD,
4387 ARM::VLD3DUPq32OddPseudo_UPD };
4388 SelectVLDDup(
N,
false,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4393 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4394 ARM::VLD4DUPd16Pseudo_UPD,
4395 ARM::VLD4DUPd32Pseudo_UPD,
4396 ARM::VLD1d64QPseudoWB_fixed };
4397 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4398 ARM::VLD4DUPq16EvenPseudo,
4399 ARM::VLD4DUPq32EvenPseudo };
4400 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4401 ARM::VLD4DUPq16OddPseudo_UPD,
4402 ARM::VLD4DUPq32OddPseudo_UPD };
4403 SelectVLDDup(
N,
false,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4408 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4409 ARM::VLD1d16wb_fixed,
4410 ARM::VLD1d32wb_fixed,
4411 ARM::VLD1d64wb_fixed };
4412 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4413 ARM::VLD1q16wb_fixed,
4414 ARM::VLD1q32wb_fixed,
4415 ARM::VLD1q64wb_fixed };
4416 SelectVLD(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4421 if (Subtarget->hasNEON()) {
4422 static const uint16_t DOpcodes[] = {
4423 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4424 ARM::VLD1q64wb_fixed};
4425 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4426 ARM::VLD2q16PseudoWB_fixed,
4427 ARM::VLD2q32PseudoWB_fixed};
4428 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4430 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4431 ARM::MVE_VLD21_8_wb};
4432 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4433 ARM::MVE_VLD21_16_wb};
4434 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4435 ARM::MVE_VLD21_32_wb};
4436 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4437 SelectMVE_VLD(
N, 2, Opcodes,
true);
4443 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4444 ARM::VLD3d16Pseudo_UPD,
4445 ARM::VLD3d32Pseudo_UPD,
4446 ARM::VLD1d64TPseudoWB_fixed};
4447 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4448 ARM::VLD3q16Pseudo_UPD,
4449 ARM::VLD3q32Pseudo_UPD };
4450 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4451 ARM::VLD3q16oddPseudo_UPD,
4452 ARM::VLD3q32oddPseudo_UPD };
4453 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4458 if (Subtarget->hasNEON()) {
4459 static const uint16_t DOpcodes[] = {
4460 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4461 ARM::VLD1d64QPseudoWB_fixed};
4462 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4463 ARM::VLD4q16Pseudo_UPD,
4464 ARM::VLD4q32Pseudo_UPD};
4465 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4466 ARM::VLD4q16oddPseudo_UPD,
4467 ARM::VLD4q32oddPseudo_UPD};
4468 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4470 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4472 ARM::MVE_VLD43_8_wb};
4473 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4475 ARM::MVE_VLD43_16_wb};
4476 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4478 ARM::MVE_VLD43_32_wb};
4479 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4480 SelectMVE_VLD(
N, 4, Opcodes,
true);
4486 if (Subtarget->hasNEON()) {
4487 static const uint16_t DOpcodes[] = {
4488 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4489 ARM::VLD1q64wb_fixed};
4490 static const uint16_t QOpcodes[] = {
4491 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4492 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4493 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4500 if (Subtarget->hasNEON()) {
4501 static const uint16_t DOpcodes[] = {
4502 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4503 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4504 static const uint16_t QOpcodes0[] = {
4505 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4506 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4507 static const uint16_t QOpcodes1[] = {
4508 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4509 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4510 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4517 if (Subtarget->hasNEON()) {
4518 static const uint16_t DOpcodes[] = {
4519 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4520 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4521 static const uint16_t QOpcodes0[] = {
4522 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4523 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4524 static const uint16_t QOpcodes1[] = {
4525 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4526 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4527 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4534 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4535 ARM::VLD2LNd16Pseudo_UPD,
4536 ARM::VLD2LNd32Pseudo_UPD };
4537 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4538 ARM::VLD2LNq32Pseudo_UPD };
4539 SelectVLDSTLane(
N,
true,
true, 2, DOpcodes, QOpcodes);
4544 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4545 ARM::VLD3LNd16Pseudo_UPD,
4546 ARM::VLD3LNd32Pseudo_UPD };
4547 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4548 ARM::VLD3LNq32Pseudo_UPD };
4549 SelectVLDSTLane(
N,
true,
true, 3, DOpcodes, QOpcodes);
4554 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4555 ARM::VLD4LNd16Pseudo_UPD,
4556 ARM::VLD4LNd32Pseudo_UPD };
4557 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4558 ARM::VLD4LNq32Pseudo_UPD };
4559 SelectVLDSTLane(
N,
true,
true, 4, DOpcodes, QOpcodes);
4564 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4565 ARM::VST1d16wb_fixed,
4566 ARM::VST1d32wb_fixed,
4567 ARM::VST1d64wb_fixed };
4568 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4569 ARM::VST1q16wb_fixed,
4570 ARM::VST1q32wb_fixed,
4571 ARM::VST1q64wb_fixed };
4572 SelectVST(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4577 if (Subtarget->hasNEON()) {
4578 static const uint16_t DOpcodes[] = {
4579 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4580 ARM::VST1q64wb_fixed};
4581 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4582 ARM::VST2q16PseudoWB_fixed,
4583 ARM::VST2q32PseudoWB_fixed};
4584 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4591 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4592 ARM::VST3d16Pseudo_UPD,
4593 ARM::VST3d32Pseudo_UPD,
4594 ARM::VST1d64TPseudoWB_fixed};
4595 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4596 ARM::VST3q16Pseudo_UPD,
4597 ARM::VST3q32Pseudo_UPD };
4598 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4599 ARM::VST3q16oddPseudo_UPD,
4600 ARM::VST3q32oddPseudo_UPD };
4601 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4606 if (Subtarget->hasNEON()) {
4607 static const uint16_t DOpcodes[] = {
4608 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4609 ARM::VST1d64QPseudoWB_fixed};
4610 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4611 ARM::VST4q16Pseudo_UPD,
4612 ARM::VST4q32Pseudo_UPD};
4613 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4614 ARM::VST4q16oddPseudo_UPD,
4615 ARM::VST4q32oddPseudo_UPD};
4616 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4623 if (Subtarget->hasNEON()) {
4624 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4625 ARM::VST1q16wb_fixed,
4626 ARM::VST1q32wb_fixed,
4627 ARM::VST1q64wb_fixed};
4628 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4629 ARM::VST1d16QPseudoWB_fixed,
4630 ARM::VST1d32QPseudoWB_fixed,
4631 ARM::VST1d64QPseudoWB_fixed };
4632 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4639 if (Subtarget->hasNEON()) {
4640 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4641 ARM::VST1d16TPseudoWB_fixed,
4642 ARM::VST1d32TPseudoWB_fixed,
4643 ARM::VST1d64TPseudoWB_fixed };
4644 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4645 ARM::VST1q16LowTPseudo_UPD,
4646 ARM::VST1q32LowTPseudo_UPD,
4647 ARM::VST1q64LowTPseudo_UPD };
4648 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4649 ARM::VST1q16HighTPseudo_UPD,
4650 ARM::VST1q32HighTPseudo_UPD,
4651 ARM::VST1q64HighTPseudo_UPD };
4652 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4659 if (Subtarget->hasNEON()) {
4660 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4661 ARM::VST1d16QPseudoWB_fixed,
4662 ARM::VST1d32QPseudoWB_fixed,
4663 ARM::VST1d64QPseudoWB_fixed };
4664 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4665 ARM::VST1q16LowQPseudo_UPD,
4666 ARM::VST1q32LowQPseudo_UPD,
4667 ARM::VST1q64LowQPseudo_UPD };
4668 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4669 ARM::VST1q16HighQPseudo_UPD,
4670 ARM::VST1q32HighQPseudo_UPD,
4671 ARM::VST1q64HighQPseudo_UPD };
4672 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4678 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4679 ARM::VST2LNd16Pseudo_UPD,
4680 ARM::VST2LNd32Pseudo_UPD };
4681 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4682 ARM::VST2LNq32Pseudo_UPD };
4683 SelectVLDSTLane(
N,
false,
true, 2, DOpcodes, QOpcodes);
4688 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4689 ARM::VST3LNd16Pseudo_UPD,
4690 ARM::VST3LNd32Pseudo_UPD };
4691 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4692 ARM::VST3LNq32Pseudo_UPD };
4693 SelectVLDSTLane(
N,
false,
true, 3, DOpcodes, QOpcodes);
4698 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4699 ARM::VST4LNd16Pseudo_UPD,
4700 ARM::VST4LNd32Pseudo_UPD };
4701 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4702 ARM::VST4LNq32Pseudo_UPD };
4703 SelectVLDSTLane(
N,
false,
true, 4, DOpcodes, QOpcodes);
4709 unsigned IntNo =
N->getConstantOperandVal(1);
4714 case Intrinsic::arm_mrrc:
4715 case Intrinsic::arm_mrrc2: {
4720 if (Subtarget->isThumb())
4721 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4723 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4726 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(2), dl));
4727 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(3), dl));
4728 Ops.
push_back(getI32Imm(
N->getConstantOperandVal(4), dl));
4733 if (Opc != ARM::MRRC2) {
4735 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4741 const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
4743 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4746 case Intrinsic::arm_ldaexd:
4747 case Intrinsic::arm_ldrexd: {
4750 SDValue MemAddr =
N->getOperand(2);
4751 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4753 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4754 unsigned NewOpc =
isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4755 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4758 std::vector<EVT> ResTys;
4760 ResTys.push_back(MVT::i32);
4761 ResTys.push_back(MVT::i32);
4763 ResTys.push_back(MVT::Untyped);
4764 ResTys.push_back(MVT::Other);
4768 CurDAG->getRegister(0, MVT::i32), Chain};
4769 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4772 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4782 CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
4783 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4784 dl, MVT::i32,
SDValue(Ld, 0), SubRegIdx);
4787 ReplaceUses(
SDValue(
N, 0), Result);
4795 CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
4796 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4797 dl, MVT::i32,
SDValue(Ld, 0), SubRegIdx);
4800 ReplaceUses(
SDValue(
N, 1), Result);
4802 ReplaceUses(
SDValue(
N, 2), OutChain);
4803 CurDAG->RemoveDeadNode(
N);
4806 case Intrinsic::arm_stlexd:
4807 case Intrinsic::arm_strexd: {
4812 SDValue MemAddr =
N->getOperand(4);
4816 const EVT ResTys[] = {MVT::i32, MVT::Other};
4818 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
4829 Ops.
push_back(CurDAG->getRegister(0, MVT::i32));
4832 bool IsRelease = IntNo == Intrinsic::arm_stlexd;
4833 unsigned NewOpc =
isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
4834 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
4836 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4839 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4845 case Intrinsic::arm_neon_vld1: {
4846 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
4847 ARM::VLD1d32, ARM::VLD1d64 };
4848 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4849 ARM::VLD1q32, ARM::VLD1q64};
4850 SelectVLD(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
4854 case Intrinsic::arm_neon_vld1x2: {
4855 static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4856 ARM::VLD1q32, ARM::VLD1q64 };
4857 static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
4858 ARM::VLD1d16QPseudo,
4859 ARM::VLD1d32QPseudo,
4860 ARM::VLD1d64QPseudo };
4861 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4865 case Intrinsic::arm_neon_vld1x3: {
4866 static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
4867 ARM::VLD1d16TPseudo,
4868 ARM::VLD1d32TPseudo,
4869 ARM::VLD1d64TPseudo };
4870 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
4871 ARM::VLD1q16LowTPseudo_UPD,
4872 ARM::VLD1q32LowTPseudo_UPD,
4873 ARM::VLD1q64LowTPseudo_UPD };
4874 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
4875 ARM::VLD1q16HighTPseudo,
4876 ARM::VLD1q32HighTPseudo,
4877 ARM::VLD1q64HighTPseudo };
4878 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4882 case Intrinsic::arm_neon_vld1x4: {
4883 static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
4884 ARM::VLD1d16QPseudo,
4885 ARM::VLD1d32QPseudo,
4886 ARM::VLD1d64QPseudo };
4887 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
4888 ARM::VLD1q16LowQPseudo_UPD,
4889 ARM::VLD1q32LowQPseudo_UPD,
4890 ARM::VLD1q64LowQPseudo_UPD };
4891 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
4892 ARM::VLD1q16HighQPseudo,
4893 ARM::VLD1q32HighQPseudo,
4894 ARM::VLD1q64HighQPseudo };
4895 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4899 case Intrinsic::arm_neon_vld2: {
4900 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
4901 ARM::VLD2d32, ARM::VLD1q64 };
4902 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
4903 ARM::VLD2q32Pseudo };
4904 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4908 case Intrinsic::arm_neon_vld3: {
4909 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
4912 ARM::VLD1d64TPseudo };
4913 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4914 ARM::VLD3q16Pseudo_UPD,
4915 ARM::VLD3q32Pseudo_UPD };
4916 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
4917 ARM::VLD3q16oddPseudo,
4918 ARM::VLD3q32oddPseudo };
4919 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4923 case Intrinsic::arm_neon_vld4: {
4924 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
4927 ARM::VLD1d64QPseudo };
4928 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
4929 ARM::VLD4q16Pseudo_UPD,
4930 ARM::VLD4q32Pseudo_UPD };
4931 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
4932 ARM::VLD4q16oddPseudo,
4933 ARM::VLD4q32oddPseudo };
4934 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4938 case Intrinsic::arm_neon_vld2dup: {
4939 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4940 ARM::VLD2DUPd32, ARM::VLD1q64 };
4941 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4942 ARM::VLD2DUPq16EvenPseudo,
4943 ARM::VLD2DUPq32EvenPseudo };
4944 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
4945 ARM::VLD2DUPq16OddPseudo,
4946 ARM::VLD2DUPq32OddPseudo };
4947 SelectVLDDup(
N,
true,
false, 2,
4948 DOpcodes, QOpcodes0, QOpcodes1);
4952 case Intrinsic::arm_neon_vld3dup: {
4953 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
4954 ARM::VLD3DUPd16Pseudo,
4955 ARM::VLD3DUPd32Pseudo,
4956 ARM::VLD1d64TPseudo };
4957 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4958 ARM::VLD3DUPq16EvenPseudo,
4959 ARM::VLD3DUPq32EvenPseudo };
4960 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
4961 ARM::VLD3DUPq16OddPseudo,
4962 ARM::VLD3DUPq32OddPseudo };
4963 SelectVLDDup(
N,
true,
false, 3,
4964 DOpcodes, QOpcodes0, QOpcodes1);
4968 case Intrinsic::arm_neon_vld4dup: {
4969 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
4970 ARM::VLD4DUPd16Pseudo,
4971 ARM::VLD4DUPd32Pseudo,
4972 ARM::VLD1d64QPseudo };
4973 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4974 ARM::VLD4DUPq16EvenPseudo,
4975 ARM::VLD4DUPq32EvenPseudo };
4976 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
4977 ARM::VLD4DUPq16OddPseudo,
4978 ARM::VLD4DUPq32OddPseudo };
4979 SelectVLDDup(
N,
true,
false, 4,
4980 DOpcodes, QOpcodes0, QOpcodes1);
4984 case Intrinsic::arm_neon_vld2lane: {
4985 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
4986 ARM::VLD2LNd16Pseudo,
4987 ARM::VLD2LNd32Pseudo };
4988 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
4989 ARM::VLD2LNq32Pseudo };
4990 SelectVLDSTLane(
N,
true,
false, 2, DOpcodes, QOpcodes);
4994 case Intrinsic::arm_neon_vld3lane: {
4995 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
4996 ARM::VLD3LNd16Pseudo,
4997 ARM::VLD3LNd32Pseudo };
4998 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
4999 ARM::VLD3LNq32Pseudo };
5000 SelectVLDSTLane(
N,
true,
false, 3, DOpcodes, QOpcodes);
5004 case Intrinsic::arm_neon_vld4lane: {
5005 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
5006 ARM::VLD4LNd16Pseudo,
5007 ARM::VLD4LNd32Pseudo };
5008 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
5009 ARM::VLD4LNq32Pseudo };
5010 SelectVLDSTLane(
N,
true,
false, 4, DOpcodes, QOpcodes);
5014 case Intrinsic::arm_neon_vst1: {
5015 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
5016 ARM::VST1d32, ARM::VST1d64 };
5017 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5018 ARM::VST1q32, ARM::VST1q64 };
5019 SelectVST(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
5023 case Intrinsic::arm_neon_vst1x2: {
5024 static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5025 ARM::VST1q32, ARM::VST1q64 };
5026 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
5027 ARM::VST1d16QPseudo,
5028 ARM::VST1d32QPseudo,
5029 ARM::VST1d64QPseudo };
5030 SelectVST(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
5034 case Intrinsic::arm_neon_vst1x3: {
5035 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
5036 ARM::VST1d16TPseudo,
5037 ARM::VST1d32TPseudo,
5038 ARM::VST1d64TPseudo };
5039 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
5040 ARM::VST1q16LowTPseudo_UPD,
5041 ARM::VST1q32LowTPseudo_UPD,
5042 ARM::VST1q64LowTPseudo_UPD };
5043 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
5044 ARM::VST1q16HighTPseudo,
5045 ARM::VST1q32HighTPseudo,
5046 ARM::VST1q64HighTPseudo };
5047 SelectVST(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5051 case Intrinsic::arm_neon_vst1x4: {
5052 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
5053 ARM::VST1d16QPseudo,
5054 ARM::VST1d32QPseudo,
5055 ARM::VST1d64QPseudo };
5056 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
5057 ARM::VST1q16LowQPseudo_UPD,
5058 ARM::VST1q32LowQPseudo_UPD,
5059 ARM::VST1q64LowQPseudo_UPD };
5060 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
5061 ARM::VST1q16HighQPseudo,
5062 ARM::VST1q32HighQPseudo,
5063 ARM::VST1q64HighQPseudo };
5064 SelectVST(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5068 case Intrinsic::arm_neon_vst2: {
5069 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
5070 ARM::VST2d32, ARM::VST1q64 };
5071 static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
5072 ARM::VST2q32Pseudo };
5073 SelectVST(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
5077 case Intrinsic::arm_neon_vst3: {
5078 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
5081 ARM::VST1d64TPseudo };
5082 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
5083 ARM::VST3q16Pseudo_UPD,
5084 ARM::VST3q32Pseudo_UPD };
5085 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
5086 ARM::VST3q16oddPseudo,
5087 ARM::VST3q32oddPseudo };
5088 SelectVST(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5092 case Intrinsic::arm_neon_vst4: {
5093 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
5096 ARM::VST1d64QPseudo };
5097 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
5098 ARM::VST4q16Pseudo_UPD,
5099 ARM::VST4q32Pseudo_UPD };
5100 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
5101 ARM::VST4q16oddPseudo,
5102 ARM::VST4q32oddPseudo };
5103 SelectVST(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5107 case Intrinsic::arm_neon_vst2lane: {
5108 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
5109 ARM::VST2LNd16Pseudo,
5110 ARM::VST2LNd32Pseudo };
5111 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
5112 ARM::VST2LNq32Pseudo };
5113 SelectVLDSTLane(
N,
false,
false, 2, DOpcodes, QOpcodes);
5117 case Intrinsic::arm_neon_vst3lane: {
5118 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
5119 ARM::VST3LNd16Pseudo,
5120 ARM::VST3LNd32Pseudo };
5121 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
5122 ARM::VST3LNq32Pseudo };
5123 SelectVLDSTLane(
N,
false,
false, 3, DOpcodes, QOpcodes);
5127 case Intrinsic::arm_neon_vst4lane: {
5128 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
5129 ARM::VST4LNd16Pseudo,
5130 ARM::VST4LNd32Pseudo };
5131 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
5132 ARM::VST4LNq32Pseudo };
5133 SelectVLDSTLane(
N,
false,
false, 4, DOpcodes, QOpcodes);
5137 case Intrinsic::arm_mve_vldr_gather_base_wb:
5138 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
5139 static const uint16_t Opcodes[] = {ARM::MVE_VLDRWU32_qi_pre,
5140 ARM::MVE_VLDRDU64_qi_pre};
5141 SelectMVE_WB(
N, Opcodes,
5142 IntNo == Intrinsic::arm_mve_vldr_gather_base_wb_predicated);
5146 case Intrinsic::arm_mve_vld2q: {
5147 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8, ARM::MVE_VLD21_8};
5148 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
5150 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
5152 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5153 SelectMVE_VLD(
N, 2, Opcodes,
false);
5157 case Intrinsic::arm_mve_vld4q: {
5158 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
5159 ARM::MVE_VLD42_8, ARM::MVE_VLD43_8};
5160 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
5163 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
5166 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5167 SelectMVE_VLD(
N, 4, Opcodes,
false);
5175 unsigned IntNo =
N->getConstantOperandVal(0);
5181 case Intrinsic::arm_neon_vcvtbfp2bf: {
5183 const SDValue &Src =
N->getOperand(1);
5186 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5187 SDValue Ops[] = { Src, Src, Pred, Reg0 };
5188 CurDAG->SelectNodeTo(
N, ARM::BF16_VCVTB, DestTy, Ops);
5193 case Intrinsic::arm_neon_vcvtfp2bf: {
5195 const SDValue &Src =
N->getOperand(1);
5197 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5198 SDValue Ops[] = { Src, Pred, Reg0 };
5199 CurDAG->SelectNodeTo(
N, ARM::BF16_VCVT, MVT::v4bf16, Ops);
5203 case Intrinsic::arm_mve_urshrl:
5204 SelectMVE_LongShift(
N, ARM::MVE_URSHRL,
true,
false);
5206 case Intrinsic::arm_mve_uqshll:
5207 SelectMVE_LongShift(
N, ARM::MVE_UQSHLL,
true,
false);
5209 case Intrinsic::arm_mve_srshrl:
5210 SelectMVE_LongShift(
N, ARM::MVE_SRSHRL,
true,
false);
5212 case Intrinsic::arm_mve_sqshll:
5213 SelectMVE_LongShift(
N, ARM::MVE_SQSHLL,
true,
false);
5215 case Intrinsic::arm_mve_uqrshll:
5216 SelectMVE_LongShift(
N, ARM::MVE_UQRSHLL,
false,
true);
5218 case Intrinsic::arm_mve_sqrshrl:
5219 SelectMVE_LongShift(
N, ARM::MVE_SQRSHRL,
false,
true);
5222 case Intrinsic::arm_mve_vadc:
5223 case Intrinsic::arm_mve_vadc_predicated:
5224 SelectMVE_VADCSBC(
N, ARM::MVE_VADC, ARM::MVE_VADCI,
true,
5225 IntNo == Intrinsic::arm_mve_vadc_predicated);
5227 case Intrinsic::arm_mve_vsbc:
5228 case Intrinsic::arm_mve_vsbc_predicated:
5229 SelectMVE_VADCSBC(
N, ARM::MVE_VSBC, ARM::MVE_VSBCI,
true,
5230 IntNo == Intrinsic::arm_mve_vsbc_predicated);
5232 case Intrinsic::arm_mve_vshlc:
5233 case Intrinsic::arm_mve_vshlc_predicated:
5234 SelectMVE_VSHLC(
N, IntNo == Intrinsic::arm_mve_vshlc_predicated);
5237 case Intrinsic::arm_mve_vmlldava:
5238 case Intrinsic::arm_mve_vmlldava_predicated: {
5239 static const uint16_t OpcodesU[] = {
5240 ARM::MVE_VMLALDAVu16, ARM::MVE_VMLALDAVu32,
5241 ARM::MVE_VMLALDAVau16, ARM::MVE_VMLALDAVau32,
5243 static const uint16_t OpcodesS[] = {
5244 ARM::MVE_VMLALDAVs16, ARM::MVE_VMLALDAVs32,
5245 ARM::MVE_VMLALDAVas16, ARM::MVE_VMLALDAVas32,
5246 ARM::MVE_VMLALDAVxs16, ARM::MVE_VMLALDAVxs32,
5247 ARM::MVE_VMLALDAVaxs16, ARM::MVE_VMLALDAVaxs32,
5248 ARM::MVE_VMLSLDAVs16, ARM::MVE_VMLSLDAVs32,
5249 ARM::MVE_VMLSLDAVas16, ARM::MVE_VMLSLDAVas32,
5250 ARM::MVE_VMLSLDAVxs16, ARM::MVE_VMLSLDAVxs32,
5251 ARM::MVE_VMLSLDAVaxs16, ARM::MVE_VMLSLDAVaxs32,
5253 SelectMVE_VMLLDAV(
N, IntNo == Intrinsic::arm_mve_vmlldava_predicated,
5254 OpcodesS, OpcodesU);
5258 case Intrinsic::arm_mve_vrmlldavha:
5259 case Intrinsic::arm_mve_vrmlldavha_predicated: {
5260 static const uint16_t OpcodesU[] = {
5261 ARM::MVE_VRMLALDAVHu32, ARM::MVE_VRMLALDAVHau32,
5263 static const uint16_t OpcodesS[] = {
5264 ARM::MVE_VRMLALDAVHs32, ARM::MVE_VRMLALDAVHas32,
5265 ARM::MVE_VRMLALDAVHxs32, ARM::MVE_VRMLALDAVHaxs32,
5266 ARM::MVE_VRMLSLDAVHs32, ARM::MVE_VRMLSLDAVHas32,
5267 ARM::MVE_VRMLSLDAVHxs32, ARM::MVE_VRMLSLDAVHaxs32,
5269 SelectMVE_VRMLLDAVH(
N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated,
5270 OpcodesS, OpcodesU);
5274 case Intrinsic::arm_mve_vidup:
5275 case Intrinsic::arm_mve_vidup_predicated: {
5276 static const uint16_t Opcodes[] = {
5277 ARM::MVE_VIDUPu8, ARM::MVE_VIDUPu16, ARM::MVE_VIDUPu32,
5279 SelectMVE_VxDUP(
N, Opcodes,
false,
5280 IntNo == Intrinsic::arm_mve_vidup_predicated);
5284 case Intrinsic::arm_mve_vddup:
5285 case Intrinsic::arm_mve_vddup_predicated: {
5286 static const uint16_t Opcodes[] = {
5287 ARM::MVE_VDDUPu8, ARM::MVE_VDDUPu16, ARM::MVE_VDDUPu32,
5289 SelectMVE_VxDUP(
N, Opcodes,
false,
5290 IntNo == Intrinsic::arm_mve_vddup_predicated);
5294 case Intrinsic::arm_mve_viwdup:
5295 case Intrinsic::arm_mve_viwdup_predicated: {
5296 static const uint16_t Opcodes[] = {
5297 ARM::MVE_VIWDUPu8, ARM::MVE_VIWDUPu16, ARM::MVE_VIWDUPu32,
5299 SelectMVE_VxDUP(
N, Opcodes,
true,
5300 IntNo == Intrinsic::arm_mve_viwdup_predicated);
5304 case Intrinsic::arm_mve_vdwdup:
5305 case Intrinsic::arm_mve_vdwdup_predicated: {
5306 static const uint16_t Opcodes[] = {
5307 ARM::MVE_VDWDUPu8, ARM::MVE_VDWDUPu16, ARM::MVE_VDWDUPu32,
5309 SelectMVE_VxDUP(
N, Opcodes,
true,
5310 IntNo == Intrinsic::arm_mve_vdwdup_predicated);
5314 case Intrinsic::arm_cde_cx1d:
5315 case Intrinsic::arm_cde_cx1da:
5316 case Intrinsic::arm_cde_cx2d:
5317 case Intrinsic::arm_cde_cx2da:
5318 case Intrinsic::arm_cde_cx3d:
5319 case Intrinsic::arm_cde_cx3da: {
5320 bool HasAccum = IntNo == Intrinsic::arm_cde_cx1da ||
5321 IntNo == Intrinsic::arm_cde_cx2da ||
5322 IntNo == Intrinsic::arm_cde_cx3da;
5326 case Intrinsic::arm_cde_cx1d:
5327 case Intrinsic::arm_cde_cx1da:
5329 Opcode = HasAccum ? ARM::CDE_CX1DA : ARM::CDE_CX1D;
5331 case Intrinsic::arm_cde_cx2d:
5332 case Intrinsic::arm_cde_cx2da:
5334 Opcode = HasAccum ? ARM::CDE_CX2DA : ARM::CDE_CX2D;
5336 case Intrinsic::arm_cde_cx3d:
5337 case Intrinsic::arm_cde_cx3da:
5339 Opcode = HasAccum ? ARM::CDE_CX3DA : ARM::CDE_CX3D;
5344 SelectCDE_CXxD(
N, Opcode, NumExtraOps, HasAccum);
5367 std::vector<SDValue> &Ops) {
5369 RegString.
split(Fields,
':');
5371 if (Fields.
size() > 1) {
5372 bool AllIntFields =
true;
5377 AllIntFields &= !
Field.trim(
"CPcp").getAsInteger(10, IntField);
5382 "Unexpected non-integer value in special register string.");
5393 auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.
lower());
5396 return TheReg->Encoding;
5408 .
Case(
"nzcvqg", 0x3)
5416 auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
5417 const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
5418 if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
5420 return (
int)(TheReg->Encoding & 0xFFF);
5429 if (Reg ==
"apsr") {
5439 if (Reg !=
"cpsr" && Reg !=
"spsr") {
5444 if (Flags.empty() || Flags ==
"all")
5449 for (
char Flag : Flags) {
5469 if (!FlagVal || (Mask & FlagVal))
5484bool ARMDAGToDAGISel::tryReadRegister(
SDNode *
N){
5485 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
5486 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5487 bool IsThumb2 = Subtarget->
isThumb2();
5490 std::vector<SDValue> Ops;
5500 if (Ops.size() == 5){
5501 Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
5502 ResTypes.
append({ MVT::i32, MVT::Other });
5504 assert(Ops.size() == 3 &&
5505 "Invalid number of fields in special register string.");
5506 Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
5507 ResTypes.
append({ MVT::i32, MVT::i32, MVT::Other });
5510 Ops.push_back(
getAL(CurDAG,
DL));
5511 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5512 Ops.push_back(
N->getOperand(0));
5513 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, ResTypes, Ops));
5517 std::string SpecialReg = RegString->getString().lower();
5520 if (BankedReg != -1) {
5521 Ops = { CurDAG->getTargetConstant(BankedReg,
DL, MVT::i32),
5522 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5525 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
5526 DL, MVT::i32, MVT::Other, Ops));
5534 .
Case(
"fpscr", ARM::VMRS)
5535 .
Case(
"fpexc", ARM::VMRS_FPEXC)
5536 .
Case(
"fpsid", ARM::VMRS_FPSID)
5537 .
Case(
"mvfr0", ARM::VMRS_MVFR0)
5538 .
Case(
"mvfr1", ARM::VMRS_MVFR1)
5539 .
Case(
"mvfr2", ARM::VMRS_MVFR2)
5540 .
Case(
"fpinst", ARM::VMRS_FPINST)
5541 .
Case(
"fpinst2", ARM::VMRS_FPINST2)
5551 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5554 CurDAG->getMachineNode(Opcode,
DL, MVT::i32, MVT::Other, Ops));
5563 if (SYSmValue == -1)
5566 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue,
DL, MVT::i32),
5567 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5570 N, CurDAG->getMachineNode(ARM::t2MRS_M,
DL, MVT::i32, MVT::Other, Ops));
5576 if (SpecialReg ==
"apsr" || SpecialReg ==
"cpsr") {
5577 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5579 ReplaceNode(
N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
5580 DL, MVT::i32, MVT::Other, Ops));
5584 if (SpecialReg ==
"spsr") {
5585 Ops = {
getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5588 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys,
DL,
5589 MVT::i32, MVT::Other, Ops));
5599bool ARMDAGToDAGISel::tryWriteRegister(
SDNode *
N){
5600 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
5601 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5602 bool IsThumb2 = Subtarget->
isThumb2();
5605 std::vector<SDValue> Ops;
5614 if (Ops.size() == 5) {
5615 Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
5616 Ops.insert(Ops.begin()+2,
N->getOperand(2));
5618 assert(Ops.size() == 3 &&
5619 "Invalid number of fields in special register string.");
5620 Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
5621 SDValue WriteValue[] = {
N->getOperand(2),
N->getOperand(3) };
5622 Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
5625 Ops.push_back(
getAL(CurDAG,
DL));
5626 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5627 Ops.push_back(
N->getOperand(0));
5629 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
5633 std::string SpecialReg = RegString->getString().lower();
5635 if (BankedReg != -1) {
5636 Ops = { CurDAG->getTargetConstant(BankedReg,
DL, MVT::i32),
N->getOperand(2),
5637 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5640 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
5641 DL, MVT::Other, Ops));
5649 .
Case(
"fpscr", ARM::VMSR)
5650 .
Case(
"fpexc", ARM::VMSR_FPEXC)
5651 .
Case(
"fpsid", ARM::VMSR_FPSID)
5652 .
Case(
"fpinst", ARM::VMSR_FPINST)
5653 .
Case(
"fpinst2", ARM::VMSR_FPINST2)
5659 Ops = {
N->getOperand(2),
getAL(CurDAG,
DL),
5660 CurDAG->getRegister(0, MVT::i32),
N->getOperand(0) };
5661 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
DL, MVT::Other, Ops));
5665 std::pair<StringRef, StringRef> Fields;
5667 std::string
Reg = Fields.first.str();
5674 if (SYSmValue == -1)
5677 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue,
DL, MVT::i32),
5678 N->getOperand(2),
getAL(CurDAG,
DL),
5679 CurDAG->getRegister(0, MVT::i32),
N->getOperand(0) };
5680 ReplaceNode(
N, CurDAG->getMachineNode(ARM::t2MSR_M,
DL, MVT::Other, Ops));
5689 Ops = { CurDAG->getTargetConstant(Mask,
DL, MVT::i32),
N->getOperand(2),
5690 getAL(CurDAG,
DL), CurDAG->getRegister(0, MVT::i32),
5692 ReplaceNode(
N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
5693 DL, MVT::Other, Ops));
5700bool ARMDAGToDAGISel::tryInlineAsm(
SDNode *
N){
5701 std::vector<SDValue> AsmNodeOperands;
5703 bool Changed =
false;
5704 unsigned NumOps =
N->getNumOperands();
5719 for(
unsigned i = 0, e =
N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
5721 AsmNodeOperands.push_back(
op);
5726 if (
const auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(i)))
5735 if (
Flag.isImmKind()) {
5737 AsmNodeOperands.push_back(
op);
5741 const unsigned NumRegs =
Flag.getNumOperandRegisters();
5745 unsigned DefIdx = 0;
5746 bool IsTiedToChangedOp =
false;
5749 if (Changed &&
Flag.isUseOperandTiedToDef(DefIdx))
5750 IsTiedToChangedOp = OpChanged[DefIdx];
5757 if (
Flag.isMemKind()) {
5759 AsmNodeOperands.push_back(
op);
5763 if (!
Flag.isRegUseKind() && !
Flag.isRegDefKind() &&
5764 !
Flag.isRegDefEarlyClobberKind())
5768 const bool HasRC =
Flag.hasRegClassConstraint(RC);
5769 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
5773 assert((i+2 < NumOps) &&
"Invalid number of operands in inline asm");
5776 Register Reg0 = cast<RegisterSDNode>(V0)->getReg();
5777 Register Reg1 = cast<RegisterSDNode>(V1)->getReg();
5781 if (
Flag.isRegDefKind() ||
Flag.isRegDefEarlyClobberKind()) {
5785 Register GPVR =
MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5786 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5789 SDNode *GU =
N->getGluedUser();
5790 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
5794 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
5796 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
5798 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
5804 Ops.push_back(
T1.getValue(1));
5805 CurDAG->UpdateNodeOperands(GU, Ops);
5812 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
5814 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
5820 Register GPVR =
MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5821 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5822 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair,
T1.getValue(1));
5831 OpChanged[OpChanged.
size() -1 ] =
true;
5833 if (IsTiedToChangedOp)
5834 Flag.setMatchingOp(DefIdx);
5836 Flag.setRegClass(ARM::GPRPairRegClassID);
5838 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
5839 Flag, dl, MVT::i32);
5841 AsmNodeOperands.push_back(PairedReg);
5848 AsmNodeOperands.push_back(Glue);
5853 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
5855 ReplaceNode(
N,
New.getNode());
5859bool ARMDAGToDAGISel::SelectInlineAsmMemoryOperand(
5861 std::vector<SDValue> &OutOps) {
5862 switch(ConstraintID) {
5865 case InlineAsm::ConstraintCode::m:
5866 case InlineAsm::ConstraintCode::o:
5867 case InlineAsm::ConstraintCode::Q:
5868 case InlineAsm::ConstraintCode::Um:
5869 case InlineAsm::ConstraintCode::Un:
5870 case InlineAsm::ConstraintCode::Uq:
5871 case InlineAsm::ConstraintCode::Us:
5872 case InlineAsm::ConstraintCode::Ut:
5873 case InlineAsm::ConstraintCode::Uv:
5874 case InlineAsm::ConstraintCode::Uy:
5878 OutOps.push_back(
Op);
5889 return new ARMDAGToDAGISel(
TM, OptLevel);
unsigned const MachineRegisterInfo * MRI
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
static bool isThumb(const MCSubtargetInfo &STI)
static unsigned getVectorShuffleOpcode(EVT VT, unsigned Opc64[3], unsigned Opc128[3])
static int getBankedRegisterMask(StringRef RegString)
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
static bool isVSTfixed(unsigned Opc)
static bool isVLDfixed(unsigned Opc)
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static std::optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
static bool shouldUseZeroOffsetLdSt(SDValue N)
static int getMClassFlagsMask(StringRef Flags)
static bool SDValueToConstBool(SDValue SDVal)
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
This file describes how to lower LLVM code to machine code.
support::ulittle16_t & Lo
support::ulittle16_t & Hi
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
An arbitrary precision integer that knows its signedness.
bool isThumb1Only() const
bool hasFPARMv8Base() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
uint64_t getZExtValue() const
This class represents an Operation in the Expression.
Container class for subtarget features.
FunctionPass class - This class is used to implement most global optimizations.
This class is used to form a handle around another node that is persistent and is updated across invo...
Base class for LoadSDNode and StoreSDNode.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
unsigned getOpcode() const
Return the opcode number for this descriptor.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MLOAD node.
This is an abstract virtual class for memory operations.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
int getNodeId() const
Return the unique node id.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string lower() const
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt32Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
float getFPImmFloat(unsigned Imm)
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOptLevel OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
CodeGenOptLevel
Code generation optimization level.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ And
Bitwise or logical AND of integers.
@ NearestTiesToEven
roundTiesToEven.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
These are IR-level optimization flags that may be propagated to SDNodes.