28 #define DEBUG_TYPE "aarch64-isel"
51 return "AArch64 Instruction Selection";
64 bool SelectInlineAsmMemoryOperand(
const SDValue &
Op,
65 unsigned ConstraintID,
66 std::vector<SDValue> &OutOps)
override;
68 bool tryMLAV64LaneV128(
SDNode *
N);
69 bool tryMULLV64LaneV128(
unsigned IntNo,
SDNode *N);
74 return SelectShiftedRegister(N,
false, Reg, Shift);
77 return SelectShiftedRegister(N,
true, Reg, Shift);
80 return SelectAddrModeIndexed7S(N, 1, Base, OffImm);
83 return SelectAddrModeIndexed7S(N, 2, Base, OffImm);
86 return SelectAddrModeIndexed7S(N, 4, Base, OffImm);
89 return SelectAddrModeIndexed7S(N, 8, Base, OffImm);
92 return SelectAddrModeIndexed7S(N, 16, Base, OffImm);
95 return SelectAddrModeIndexed(N, 1, Base, OffImm);
98 return SelectAddrModeIndexed(N, 2, Base, OffImm);
101 return SelectAddrModeIndexed(N, 4, Base, OffImm);
104 return SelectAddrModeIndexed(N, 8, Base, OffImm);
107 return SelectAddrModeIndexed(N, 16, Base, OffImm);
110 return SelectAddrModeUnscaled(N, 1, Base, OffImm);
113 return SelectAddrModeUnscaled(N, 2, Base, OffImm);
116 return SelectAddrModeUnscaled(N, 4, Base, OffImm);
119 return SelectAddrModeUnscaled(N, 8, Base, OffImm);
122 return SelectAddrModeUnscaled(N, 16, Base, OffImm);
128 return SelectAddrModeWRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
134 return SelectAddrModeXRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
148 const unsigned SubRegs[]);
150 void SelectTable(
SDNode *N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
152 bool tryIndexedLoad(
SDNode *N);
154 void SelectLoad(
SDNode *N,
unsigned NumVecs,
unsigned Opc,
156 void SelectPostLoad(
SDNode *N,
unsigned NumVecs,
unsigned Opc,
158 void SelectLoadLane(
SDNode *N,
unsigned NumVecs,
unsigned Opc);
159 void SelectPostLoadLane(
SDNode *N,
unsigned NumVecs,
unsigned Opc);
161 void SelectStore(
SDNode *N,
unsigned NumVecs,
unsigned Opc);
162 void SelectPostStore(
SDNode *N,
unsigned NumVecs,
unsigned Opc);
163 void SelectStoreLane(
SDNode *N,
unsigned NumVecs,
unsigned Opc);
164 void SelectPostStoreLane(
SDNode *N,
unsigned NumVecs,
unsigned Opc);
166 bool tryBitfieldExtractOp(
SDNode *N);
167 bool tryBitfieldExtractOpFromSExt(
SDNode *N);
168 bool tryBitfieldInsertOp(
SDNode *N);
169 bool tryBitfieldInsertInZeroOp(
SDNode *N);
171 bool tryReadRegister(
SDNode *N);
172 bool tryWriteRegister(
SDNode *N);
175 #include "AArch64GenDAGISel.inc"
178 bool SelectShiftedRegister(
SDValue N,
bool AllowROR,
SDValue &Reg,
180 bool SelectAddrModeIndexed7S(
SDValue N,
unsigned Size,
SDValue &Base,
182 bool SelectAddrModeIndexed(
SDValue N,
unsigned Size,
SDValue &Base,
184 bool SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
SDValue &Base,
192 bool isWorthFolding(
SDValue V)
const;
193 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
196 template<
unsigned RegW
idth>
198 return SelectCVTFixedPosOperand(N, FixedPos, RegWidth);
201 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
203 void SelectCMP_SWAP(
SDNode *N);
212 Imm =
C->getZExtValue();
233 bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
234 const SDValue &
Op,
unsigned ConstraintID, std::vector<SDValue> &OutOps) {
235 switch(ConstraintID) {
244 OutOps.push_back(Op);
260 if (!isa<ConstantSDNode>(N.
getNode()))
263 uint64_t Immed = cast<ConstantSDNode>(N.
getNode())->getZExtValue();
266 if (Immed >> 12 == 0) {
268 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
276 Val = CurDAG->getTargetConstant(Immed, dl,
MVT::i32);
277 Shift = CurDAG->getTargetConstant(ShVal, dl,
MVT::i32);
283 bool AArch64DAGToDAGISel::SelectNegArithImmed(
SDValue N,
SDValue &Val,
290 if (!isa<ConstantSDNode>(N.
getNode()))
294 uint64_t Immed = cast<ConstantSDNode>(N.
getNode())->getZExtValue();
305 Immed = ~Immed + 1ULL;
306 if (Immed & 0xFFFFFFFFFF000000ULL)
309 Immed &= 0xFFFFFFULL;
310 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(N),
MVT::i32), Val,
332 bool AArch64DAGToDAGISel::isWorthFolding(
SDValue V)
const {
343 bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
353 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
357 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(N),
MVT::i32);
358 return isWorthFolding(N);
372 SrcVT = cast<VTSDNode>(N.
getOperand(1))->getVT();
376 if (!IsLoadStore && SrcVT ==
MVT::i8)
378 else if (!IsLoadStore && SrcVT ==
MVT::i16)
388 if (!IsLoadStore && SrcVT ==
MVT::i8)
390 else if (!IsLoadStore && SrcVT ==
MVT::i16)
434 LaneIdx = DLidx->
getSExtValue() + EVidx->getSExtValue();
443 SDValue &LaneOp,
int &LaneIdx) {
457 bool AArch64DAGToDAGISel::tryMLAV64LaneV128(
SDNode *N) {
477 SDValue Ops[] = { Op0, MLAOp1, MLAOp2, LaneIdxVal };
479 unsigned MLAOpc = ~0U;
485 MLAOpc = AArch64::MLAv4i16_indexed;
488 MLAOpc = AArch64::MLAv8i16_indexed;
491 MLAOpc = AArch64::MLAv2i32_indexed;
494 MLAOpc = AArch64::MLAv4i32_indexed;
498 ReplaceNode(N, CurDAG->getMachineNode(MLAOpc, dl, N->
getValueType(0), Ops));
502 bool AArch64DAGToDAGISel::tryMULLV64LaneV128(
unsigned IntNo,
SDNode *N) {
514 SDValue Ops[] = { SMULLOp0, SMULLOp1, LaneIdxVal };
516 unsigned SMULLOpc = ~0U;
518 if (IntNo == Intrinsic::aarch64_neon_smull) {
523 SMULLOpc = AArch64::SMULLv4i16_indexed;
526 SMULLOpc = AArch64::SMULLv2i32_indexed;
529 }
else if (IntNo == Intrinsic::aarch64_neon_umull) {
534 SMULLOpc = AArch64::UMULLv4i16_indexed;
537 SMULLOpc = AArch64::UMULLv2i32_indexed;
543 ReplaceNode(N, CurDAG->getMachineNode(SMULLOpc, dl, N->
getValueType(0), Ops));
565 bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
567 unsigned ShiftVal = 0;
605 return isWorthFolding(N);
632 bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(
SDValue N,
unsigned Size,
639 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
640 Base = CurDAG->getTargetFrameIndex(FI, TLI->
getPointerTy(DL));
641 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
648 if (CurDAG->isBaseWithConstantOffset(N)) {
650 int64_t RHSC = RHS->getSExtValue();
651 unsigned Scale =
Log2_32(Size);
652 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(0x40 << Scale) &&
653 RHSC < (0x40 << Scale)) {
656 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
657 Base = CurDAG->getTargetFrameIndex(FI, TLI->
getPointerTy(DL));
659 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl,
MVT::i64);
670 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
677 bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
680 const DataLayout &DL = CurDAG->getDataLayout();
683 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
684 Base = CurDAG->getTargetFrameIndex(FI, TLI->
getPointerTy(DL));
685 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
700 if (Alignment == 0 && Ty->
isSized())
703 if (Alignment >= Size)
707 if (CurDAG->isBaseWithConstantOffset(N)) {
709 int64_t RHSC = (int64_t)RHS->getZExtValue();
710 unsigned Scale =
Log2_32(Size);
711 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
714 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
715 Base = CurDAG->getTargetFrameIndex(FI, TLI->
getPointerTy(DL));
717 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl,
MVT::i64);
725 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
733 OffImm = CurDAG->getTargetConstant(0, dl,
MVT::i64);
742 bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
745 if (!CurDAG->isBaseWithConstantOffset(N))
748 int64_t RHSC = RHS->getSExtValue();
750 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 &&
751 RHSC < (0x1000 <<
Log2_32(Size)))
753 if (RHSC >= -256 && RHSC < 256) {
756 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
758 Base = CurDAG->getTargetFrameIndex(
761 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(N),
MVT::i64);
774 TargetOpcode::INSERT_SUBREG, dl,
MVT::i64, ImpDef, N, SubReg);
780 bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
785 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
800 SignExtend = CurDAG->getTargetConstant(0, dl,
MVT::i32);
803 unsigned LegalShiftVal =
Log2_32(Size);
804 unsigned ShiftVal = CSD->getZExtValue();
806 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
809 return isWorthFolding(N);
812 bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
824 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
832 if (!isa<MemSDNode>(*UI))
837 bool IsExtendedRegisterWorthFolding = isWorthFolding(N);
841 SelectExtendedSHL(RHS, Size,
true, Offset, SignExtend)) {
843 DoShift = CurDAG->getTargetConstant(
true, dl,
MVT::i32);
849 SelectExtendedSHL(LHS, Size,
true, Offset, SignExtend)) {
851 DoShift = CurDAG->getTargetConstant(
true, dl,
MVT::i32);
856 DoShift = CurDAG->getTargetConstant(
false, dl,
MVT::i32);
860 if (IsExtendedRegisterWorthFolding &&
867 if (isWorthFolding(LHS))
872 if (IsExtendedRegisterWorthFolding &&
879 if (isWorthFolding(RHS))
891 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
894 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
896 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
897 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
901 bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
916 if (!isa<MemSDNode>(*UI))
931 if (isa<ConstantSDNode>(RHS)) {
932 int64_t ImmOff = (int64_t)cast<ConstantSDNode>(RHS)->getZExtValue();
933 unsigned Scale =
Log2_32(Size);
937 if ((ImmOff % Size == 0 && ImmOff >= 0 && ImmOff < (0x1000 << Scale)) ||
943 CurDAG->getMachineNode(AArch64::MOVi64imm, DL,
MVT::i64, Ops);
950 bool IsExtendedRegisterWorthFolding = isWorthFolding(N);
954 SelectExtendedSHL(RHS, Size,
false, Offset, SignExtend)) {
956 DoShift = CurDAG->getTargetConstant(
true, DL,
MVT::i32);
962 SelectExtendedSHL(LHS, Size,
false, Offset, SignExtend)) {
964 DoShift = CurDAG->getTargetConstant(
true, DL,
MVT::i32);
971 SignExtend = CurDAG->getTargetConstant(
false, DL,
MVT::i32);
972 DoShift = CurDAG->getTargetConstant(
false, DL,
MVT::i32);
978 static const unsigned RegClassIDs[] = {
979 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
980 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
981 AArch64::dsub2, AArch64::dsub3};
983 return createTuple(Regs, RegClassIDs, SubRegs);
987 static const unsigned RegClassIDs[] = {
988 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
989 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
990 AArch64::qsub2, AArch64::qsub3};
992 return createTuple(Regs, RegClassIDs, SubRegs);
996 const unsigned RegClassIDs[],
997 const unsigned SubRegs[]) {
1000 if (Regs.
size() == 1)
1011 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2], DL,
MVT::i32));
1014 for (
unsigned i = 0;
i < Regs.
size(); ++
i) {
1020 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
MVT::Untyped, Ops);
1024 void AArch64DAGToDAGISel::SelectTable(
SDNode *N,
unsigned NumVecs,
unsigned Opc,
1029 unsigned ExtOff = isExt;
1032 unsigned Vec0Off = ExtOff + 1;
1034 N->
op_begin() + Vec0Off + NumVecs);
1035 SDValue RegSeq = createQTuple(Regs);
1042 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1045 bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *N) {
1057 unsigned Opcode = 0;
1060 bool InsertTo64 =
false;
1062 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1065 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1067 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1069 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1078 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1080 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1082 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1091 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1093 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1095 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1102 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1104 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1106 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1108 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1118 SDNode *Res = CurDAG->getMachineNode(Opcode, dl,
MVT::i64, DstVT,
1125 SDValue(CurDAG->getMachineNode(
1126 AArch64::SUBREG_TO_REG, dl,
MVT::i64,
1127 CurDAG->getTargetConstant(0, dl,
MVT::i64), LoadedVal,
1132 ReplaceUses(
SDValue(N, 0), LoadedVal);
1135 CurDAG->RemoveDeadNode(N);
1139 void AArch64DAGToDAGISel::SelectLoad(
SDNode *N,
unsigned NumVecs,
unsigned Opc,
1140 unsigned SubRegIdx) {
1150 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1152 for (
unsigned i = 0; i < NumVecs; ++
i)
1154 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1160 MemOp[0] = cast<MemIntrinsicSDNode>(
N)->getMemOperand();
1161 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
1163 CurDAG->RemoveDeadNode(N);
1166 void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *N,
unsigned NumVecs,
1167 unsigned Opc,
unsigned SubRegIdx) {
1179 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1187 ReplaceUses(
SDValue(N, 0), SuperReg);
1189 for (
unsigned i = 0; i < NumVecs; ++
i)
1191 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1195 CurDAG->RemoveDeadNode(N);
1198 void AArch64DAGToDAGISel::SelectStore(
SDNode *N,
unsigned NumVecs,
1206 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1213 MemOp[0] = cast<MemIntrinsicSDNode>(
N)->getMemOperand();
1214 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
1219 void AArch64DAGToDAGISel::SelectPostStore(
SDNode *N,
unsigned NumVecs,
1229 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1235 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1257 SDValue(DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, WideTy), 0);
1258 return DAG.getTargetInsertSubreg(AArch64::dsub, DL, WideTy, Undef, V64Reg);
1275 void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *N,
unsigned NumVecs,
1288 SDValue RegSeq = createQTuple(Regs);
1293 cast<ConstantSDNode>(N->
getOperand(NumVecs + 2))->getZExtValue();
1295 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl,
MVT::i64),
1297 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1300 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
1301 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
1302 AArch64::qsub2, AArch64::qsub3 };
1303 for (
unsigned i = 0; i < NumVecs; ++
i) {
1304 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
1307 ReplaceUses(
SDValue(N, i), NV);
1311 CurDAG->RemoveDeadNode(N);
1314 void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *N,
unsigned NumVecs,
1327 SDValue RegSeq = createQTuple(Regs);
1333 cast<ConstantSDNode>(N->
getOperand(NumVecs + 1))->getZExtValue();
1336 CurDAG->getTargetConstant(LaneNo, dl,
1341 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1352 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
1353 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
1354 AArch64::qsub2, AArch64::qsub3 };
1355 for (
unsigned i = 0; i < NumVecs; ++
i) {
1356 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
1360 ReplaceUses(
SDValue(N, i), NV);
1366 CurDAG->RemoveDeadNode(N);
1369 void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *N,
unsigned NumVecs,
1382 SDValue RegSeq = createQTuple(Regs);
1385 cast<ConstantSDNode>(N->
getOperand(NumVecs + 2))->getZExtValue();
1387 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl,
MVT::i64),
1393 MemOp[0] = cast<MemIntrinsicSDNode>(
N)->getMemOperand();
1394 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
1399 void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *N,
unsigned NumVecs,
1412 SDValue RegSeq = createQTuple(Regs);
1418 cast<ConstantSDNode>(N->
getOperand(NumVecs + 1))->getZExtValue();
1420 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl,
MVT::i64),
1424 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1428 MemOp[0] = cast<MemIntrinsicSDNode>(
N)->getMemOperand();
1429 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
1436 unsigned &LSB,
unsigned &MSB,
1437 unsigned NumberOfIgnoredLowBits,
1438 bool BiggerPattern) {
1440 "N must be a AND operation to call this function");
1448 "Type checking must have been done before calling this function");
1458 uint64_t AndImm = 0;
1466 AndImm |= (1 << NumberOfIgnoredLowBits) - 1;
1469 if (AndImm & (AndImm + 1))
1472 bool ClampMSB =
false;
1473 uint64_t SrlImm = 0;
1492 }
else if (BiggerPattern) {
1504 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
1506 <<
": Found large shift immediate, this should not happen\n"));
1511 MSB = SrlImm + (VT ==
MVT::i32 ? countTrailingOnes<uint32_t>(AndImm)
1512 : countTrailingOnes<uint64_t>(AndImm)) -
1519 MSB = MSB > 31 ? 31 : MSB;
1521 Opc = VT ==
MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
1526 SDValue &Opd0,
unsigned &Immr,
1533 "Type checking must have been done before calling this function");
1539 BitWidth = VT.getSizeInBits();
1547 unsigned Width = cast<VTSDNode>(N->
getOperand(1))->getVT().getSizeInBits();
1548 if (ShiftImm + Width > BitWidth)
1551 Opc = (VT ==
MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
1554 Imms = ShiftImm + Width - 1;
1578 uint64_t AndMask = 0;
1584 uint64_t SrlImm = 0;
1590 if (BitWide &&
isMask_64(AndMask >> SrlImm)) {
1592 Opc = AArch64::UBFMWri;
1594 Opc = AArch64::UBFMXri;
1597 MSB = BitWide + SrlImm - 1;
1605 unsigned &Immr,
unsigned &Imms,
1606 bool BiggerPattern) {
1608 "N must be a SHR/SRA operation to call this function");
1616 "Type checking must have been done before calling this function");
1623 uint64_t ShlImm = 0;
1624 uint64_t TruncBits = 0;
1637 }
else if (BiggerPattern) {
1649 <<
": Found large shift immediate, this should not happen\n"));
1653 uint64_t SrlImm = 0;
1658 "bad amount in shift node!");
1659 int immr = SrlImm - ShlImm;
1670 bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *N) {
1686 unsigned Immr = ShiftImm;
1688 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1689 CurDAG->getTargetConstant(Imms, dl, VT)};
1690 CurDAG->SelectNodeTo(N, AArch64::SBFMXri, VT, Ops);
1695 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
1696 unsigned NumberOfIgnoredLowBits = 0,
1697 bool BiggerPattern =
false) {
1708 NumberOfIgnoredLowBits, BiggerPattern);
1721 case AArch64::SBFMWri:
1722 case AArch64::UBFMWri:
1723 case AArch64::SBFMXri:
1724 case AArch64::UBFMXri:
1735 bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *N) {
1736 unsigned Opc, Immr, Imms;
1746 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT ==
MVT::i32) {
1747 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl,
MVT::i64),
1748 CurDAG->getTargetConstant(Imms, dl,
MVT::i64)};
1751 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl,
MVT::i32);
1752 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
1757 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1758 CurDAG->getTargetConstant(Imms, dl, VT)};
1759 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
1768 unsigned NumberOfIgnoredHighBits,
EVT VT) {
1770 "i32 or i64 mask type expected!");
1771 unsigned BitWidth = VT.
getSizeInBits() - NumberOfIgnoredHighBits;
1773 APInt SignificantDstMask =
APInt(BitWidth, DstMask);
1774 APInt SignificantBitsToBeInserted = BitsToBeInserted.
zextOrTrunc(BitWidth);
1776 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
1777 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnesValue();
1802 UsefulBits &=
APInt(UsefulBits.getBitWidth(), Imm);
1807 uint64_t Imm, uint64_t MSB,
1810 APInt OpUsefulBits(UsefulBits);
1814 OpUsefulBits = OpUsefulBits.
shl(MSB - Imm + 1);
1819 OpUsefulBits = OpUsefulBits.
shl(Imm);
1821 OpUsefulBits = OpUsefulBits.
shl(MSB + 1);
1824 OpUsefulBits = OpUsefulBits.
shl(OpUsefulBits.
getBitWidth() - Imm);
1830 UsefulBits &= OpUsefulBits;
1845 uint64_t ShiftTypeAndValue =
1854 Mask = Mask.
shl(ShiftAmt);
1856 Mask = Mask.
lshr(ShiftAmt);
1862 Mask = Mask.
lshr(ShiftAmt);
1864 Mask = Mask.
shl(ShiftAmt);
1878 APInt OpUsefulBits(UsefulBits);
1889 uint64_t Width = MSB - Imm + 1;
1892 OpUsefulBits = OpUsefulBits.
shl(Width);
1897 Mask = ResultUsefulBits & OpUsefulBits;
1903 Mask |= (ResultUsefulBits & ~OpUsefulBits);
1906 uint64_t Width = MSB + 1;
1909 OpUsefulBits = OpUsefulBits.
shl(Width);
1911 OpUsefulBits = OpUsefulBits.
shl(LSB);
1915 Mask = ResultUsefulBits & OpUsefulBits;
1920 Mask |= (ResultUsefulBits & ~OpUsefulBits);
1937 case AArch64::ANDSWri:
1938 case AArch64::ANDSXri:
1939 case AArch64::ANDWri:
1940 case AArch64::ANDXri:
1944 case AArch64::UBFMWri:
1945 case AArch64::UBFMXri:
1948 case AArch64::ORRWrs:
1949 case AArch64::ORRXrs:
1954 case AArch64::BFMWri:
1955 case AArch64::BFMXri:
1958 case AArch64::STRBBui:
1959 case AArch64::STURBBi:
1965 case AArch64::STRHHui:
1966 case AArch64::STURHHi:
1981 UsefulBits =
APInt(Bitwidth, 0);
1990 UsersUsefulBits |= UsefulBitsForUse;
1995 UsefulBits &= UsersUsefulBits;
2008 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2011 if (ShlAmount > 0) {
2014 UBFMOpc, dl, VT, Op,
2019 assert(ShlAmount < 0 &&
"expected right shift");
2020 int ShrAmount = -ShlAmount;
2033 SDValue &Src,
int &ShiftAmount,
2038 assert(BitWidth == 32 || BitWidth == 64);
2040 APInt KnownZero, KnownOne;
2045 uint64_t NonZeroBits = (~KnownZero).getZExtValue();
2051 assert((~
APInt(BitWidth, AndImm) & ~KnownZero) == 0);
2076 if (ShlImm - ShiftAmount != 0 && !BiggerPattern)
2120 APInt KnownZero, KnownOne;
2125 uint64_t NotKnownZero = (~KnownZero).getZExtValue();
2132 if ((OrImm & NotKnownZero) != 0) {
2143 unsigned ImmR = (BitWidth - LSB) % BitWidth;
2144 unsigned ImmS = Width - 1;
2150 bool IsBFI = LSB != 0;
2151 uint64_t BFIImm = OrImm >> LSB;
2155 unsigned OrChunks = 0, BFIChunks = 0;
2156 for (
unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
2157 if (((OrImm >> Shift) & 0xFFFF) != 0)
2159 if (((BFIImm >> Shift) & 0xFFFF) != 0)
2162 if (BFIChunks > OrChunks)
2168 unsigned MOVIOpc = VT ==
MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
2176 unsigned Opc = (VT ==
MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2215 for (
int I = 0;
I < 4; ++
I) {
2218 unsigned ImmR, ImmS;
2219 bool BiggerPattern =
I / 2;
2228 NumberOfIgnoredLowBits, BiggerPattern)) {
2231 if ((BFXOpc != AArch64::UBFMXri && VT ==
MVT::i64) ||
2232 (BFXOpc != AArch64::UBFMWri && VT ==
MVT::i32))
2237 Width = ImmS - ImmR + 1;
2248 Src, DstLSB, Width)) {
2249 ImmR = (BitWidth - DstLSB) % BitWidth;
2262 APInt KnownZero, KnownOne;
2267 APInt BitsToBeInserted =
2270 if ((BitsToBeInserted & ~KnownZero) != 0)
2288 unsigned Opc = (VT ==
MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2296 uint64_t Mask0Imm, Mask1Imm;
2302 APInt(BitWidth, Mask0Imm) == ~
APInt(BitWidth, Mask1Imm) &&
2321 unsigned ShiftOpc = (VT ==
MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
2327 unsigned ImmR = (BitWidth - LSB) % BitWidth;
2328 unsigned ImmS = Width - 1;
2334 unsigned Opc = (VT ==
MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2342 bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *N) {
2351 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->
getValueType(0));
2364 bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *N) {
2375 Op0, DstLSB, Width))
2381 unsigned ImmS = Width - 1;
2384 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT),
2385 CurDAG->getTargetConstant(ImmS, DL, VT)};
2386 unsigned Opc = (VT ==
MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
2387 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2392 AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
2393 unsigned RegWidth) {
2396 FVal = CN->getValueAPF();
2397 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(N)) {
2400 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
2405 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
2424 if (!IsExact || !
IntVal.isPowerOf2())
return false;
2425 unsigned FBits =
IntVal.logBase2();
2429 if (FBits == 0 || FBits > RegWidth)
return false;
2431 FixedPos = CurDAG->getTargetConstant(FBits,
SDLoc(N),
MVT::i32);
2440 RegString.
split(Fields,
':');
2442 if (Fields.
size() == 1)
2446 &&
"Invalid number of fields in read register string");
2449 bool AllIntFields =
true;
2453 AllIntFields &= !
Field.getAsInteger(10, IntField);
2454 Ops.push_back(IntField);
2458 "Unexpected non-integer value in special register string.");
2462 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
2463 (Ops[3] << 3) | (Ops[4]);
2470 bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *N) {
2477 ReplaceNode(N, CurDAG->getMachineNode(
2479 CurDAG->getTargetConstant(Reg, DL,
MVT::i32),
2487 if (TheReg && TheReg->Readable &&
2488 TheReg->haveFeatures(Subtarget->getFeatureBits()))
2494 ReplaceNode(N, CurDAG->getMachineNode(
2496 CurDAG->getTargetConstant(Reg, DL,
MVT::i32),
2508 bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *N) {
2516 N, CurDAG->getMachineNode(AArch64::MSR, DL,
MVT::Other,
2517 CurDAG->getTargetConstant(Reg, DL,
MVT::i32),
2527 auto PMapper = AArch64PState::lookupPStateByName(RegString->getString());;
2530 &&
"Expected a constant integer expression.");
2531 unsigned Reg = PMapper->Encoding;
2532 uint64_t Immed = cast<ConstantSDNode>(N->
getOperand(2))->getZExtValue();
2534 if (Reg == AArch64PState::PAN || Reg == AArch64PState::UAO) {
2535 assert(Immed < 2 &&
"Bad imm");
2536 State = AArch64::MSRpstateImm1;
2538 assert(Immed < 16 &&
"Bad imm");
2539 State = AArch64::MSRpstateImm4;
2541 ReplaceNode(N, CurDAG->getMachineNode(
2543 CurDAG->getTargetConstant(Reg, DL,
MVT::i32),
2544 CurDAG->getTargetConstant(Immed, DL,
MVT::i16),
2553 if (TheReg && TheReg->Writeable &&
2554 TheReg->haveFeatures(Subtarget->getFeatureBits()))
2555 Reg = TheReg->Encoding;
2559 ReplaceNode(N, CurDAG->getMachineNode(
2561 CurDAG->getTargetConstant(Reg, DL,
MVT::i32),
2570 void AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *N) {
2572 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
2574 Opcode = AArch64::CMP_SWAP_8;
2576 Opcode = AArch64::CMP_SWAP_16;
2578 Opcode = AArch64::CMP_SWAP_32;
2580 Opcode = AArch64::CMP_SWAP_64;
2587 SDNode *CmpSwap = CurDAG->getMachineNode(
2592 MemOp[0] = cast<MemSDNode>(
N)->getMemOperand();
2593 cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1);
2597 CurDAG->RemoveDeadNode(N);
2621 SelectCMP_SWAP(Node);
2625 if (tryReadRegister(Node))
2630 if (tryWriteRegister(Node))
2635 if (tryMLAV64LaneV128(Node))
2642 if (tryIndexedLoad(Node))
2651 if (tryBitfieldExtractOp(Node))
2653 if (tryBitfieldInsertInZeroOp(Node))
2658 if (tryBitfieldExtractOpFromSExt(Node))
2663 if (tryBitfieldInsertOp(Node))
2691 SubReg = AArch64::dsub;
2694 SubReg = AArch64::ssub;
2697 SubReg = AArch64::hsub;
2702 SDValue Extract = CurDAG->getTargetExtractSubreg(SubReg,
SDLoc(Node), VT,
2704 DEBUG(
dbgs() <<
"ISEL: Custom selection!\n=> ");
2707 ReplaceNode(Node, Extract.
getNode());
2716 SDValue New = CurDAG->getCopyFromReg(
2717 CurDAG->getEntryNode(),
SDLoc(Node), AArch64::WZR,
MVT::i32);
2718 ReplaceNode(Node, New.
getNode());
2721 SDValue New = CurDAG->getCopyFromReg(
2722 CurDAG->getEntryNode(),
SDLoc(Node), AArch64::XZR,
MVT::i64);
2723 ReplaceNode(Node, New.
getNode());
2732 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
2735 SDValue TFI = CurDAG->getTargetFrameIndex(
2739 CurDAG->getTargetConstant(Shifter, DL,
MVT::i32) };
2740 CurDAG->SelectNodeTo(Node, AArch64::ADDXri,
MVT::i64, Ops);
2744 unsigned IntNo = cast<ConstantSDNode>(Node->
getOperand(1))->getZExtValue();
2748 case Intrinsic::aarch64_ldaxp:
2749 case Intrinsic::aarch64_ldxp: {
2751 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
2761 MemOp[0] = cast<MemIntrinsicSDNode>(Node)->getMemOperand();
2762 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
2763 ReplaceNode(Node, Ld);
2766 case Intrinsic::aarch64_stlxp:
2767 case Intrinsic::aarch64_stxp: {
2769 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
2777 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
2782 MemOp[0] = cast<MemIntrinsicSDNode>(Node)->getMemOperand();
2783 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
2785 ReplaceNode(Node, St);
2788 case Intrinsic::aarch64_neon_ld1x2:
2790 SelectLoad(Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
2793 SelectLoad(Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
2796 SelectLoad(Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
2799 SelectLoad(Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
2802 SelectLoad(Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
2805 SelectLoad(Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
2808 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
2811 SelectLoad(Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
2815 case Intrinsic::aarch64_neon_ld1x3:
2817 SelectLoad(Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
2820 SelectLoad(Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
2823 SelectLoad(Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
2826 SelectLoad(Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
2829 SelectLoad(Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
2832 SelectLoad(Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
2835 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
2838 SelectLoad(Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
2842 case Intrinsic::aarch64_neon_ld1x4:
2844 SelectLoad(Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
2847 SelectLoad(Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
2850 SelectLoad(Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
2853 SelectLoad(Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
2856 SelectLoad(Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
2859 SelectLoad(Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
2862 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
2865 SelectLoad(Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
2869 case Intrinsic::aarch64_neon_ld2:
2871 SelectLoad(Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
2874 SelectLoad(Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
2877 SelectLoad(Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
2880 SelectLoad(Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
2883 SelectLoad(Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
2886 SelectLoad(Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
2889 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
2892 SelectLoad(Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
2896 case Intrinsic::aarch64_neon_ld3:
2898 SelectLoad(Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
2901 SelectLoad(Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
2904 SelectLoad(Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
2907 SelectLoad(Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
2910 SelectLoad(Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
2913 SelectLoad(Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
2916 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
2919 SelectLoad(Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
2923 case Intrinsic::aarch64_neon_ld4:
2925 SelectLoad(Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
2928 SelectLoad(Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
2931 SelectLoad(Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
2934 SelectLoad(Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
2937 SelectLoad(Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
2940 SelectLoad(Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
2943 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
2946 SelectLoad(Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
2950 case Intrinsic::aarch64_neon_ld2r:
2952 SelectLoad(Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
2955 SelectLoad(Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
2958 SelectLoad(Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
2961 SelectLoad(Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
2964 SelectLoad(Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
2967 SelectLoad(Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
2970 SelectLoad(Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
2973 SelectLoad(Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
2977 case Intrinsic::aarch64_neon_ld3r:
2979 SelectLoad(Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
2982 SelectLoad(Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
2985 SelectLoad(Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
2988 SelectLoad(Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
2991 SelectLoad(Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
2994 SelectLoad(Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
2997 SelectLoad(Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
3000 SelectLoad(Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
3004 case Intrinsic::aarch64_neon_ld4r:
3006 SelectLoad(Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
3009 SelectLoad(Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
3012 SelectLoad(Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
3015 SelectLoad(Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
3018 SelectLoad(Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
3021 SelectLoad(Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
3024 SelectLoad(Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
3027 SelectLoad(Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
3031 case Intrinsic::aarch64_neon_ld2lane:
3033 SelectLoadLane(Node, 2, AArch64::LD2i8);
3037 SelectLoadLane(Node, 2, AArch64::LD2i16);
3041 SelectLoadLane(Node, 2, AArch64::LD2i32);
3045 SelectLoadLane(Node, 2, AArch64::LD2i64);
3049 case Intrinsic::aarch64_neon_ld3lane:
3051 SelectLoadLane(Node, 3, AArch64::LD3i8);
3055 SelectLoadLane(Node, 3, AArch64::LD3i16);
3059 SelectLoadLane(Node, 3, AArch64::LD3i32);
3063 SelectLoadLane(Node, 3, AArch64::LD3i64);
3067 case Intrinsic::aarch64_neon_ld4lane:
3069 SelectLoadLane(Node, 4, AArch64::LD4i8);
3073 SelectLoadLane(Node, 4, AArch64::LD4i16);
3077 SelectLoadLane(Node, 4, AArch64::LD4i32);
3081 SelectLoadLane(Node, 4, AArch64::LD4i64);
3088 unsigned IntNo = cast<ConstantSDNode>(Node->
getOperand(0))->getZExtValue();
3092 case Intrinsic::aarch64_neon_tbl2:
3093 SelectTable(Node, 2,
3094 VT ==
MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
3097 case Intrinsic::aarch64_neon_tbl3:
3098 SelectTable(Node, 3, VT ==
MVT::v8i8 ? AArch64::TBLv8i8Three
3099 : AArch64::TBLv16i8Three,
3102 case Intrinsic::aarch64_neon_tbl4:
3103 SelectTable(Node, 4, VT ==
MVT::v8i8 ? AArch64::TBLv8i8Four
3104 : AArch64::TBLv16i8Four,
3107 case Intrinsic::aarch64_neon_tbx2:
3108 SelectTable(Node, 2,
3109 VT ==
MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
3112 case Intrinsic::aarch64_neon_tbx3:
3113 SelectTable(Node, 3, VT ==
MVT::v8i8 ? AArch64::TBXv8i8Three
3114 : AArch64::TBXv16i8Three,
3117 case Intrinsic::aarch64_neon_tbx4:
3118 SelectTable(Node, 4, VT ==
MVT::v8i8 ? AArch64::TBXv8i8Four
3119 : AArch64::TBXv16i8Four,
3122 case Intrinsic::aarch64_neon_smull:
3123 case Intrinsic::aarch64_neon_umull:
3124 if (tryMULLV64LaneV128(IntNo, Node))
3131 unsigned IntNo = cast<ConstantSDNode>(Node->
getOperand(1))->getZExtValue();
3137 case Intrinsic::aarch64_neon_st1x2: {
3139 SelectStore(Node, 2, AArch64::ST1Twov8b);
3142 SelectStore(Node, 2, AArch64::ST1Twov16b);
3145 SelectStore(Node, 2, AArch64::ST1Twov4h);
3148 SelectStore(Node, 2, AArch64::ST1Twov8h);
3151 SelectStore(Node, 2, AArch64::ST1Twov2s);
3154 SelectStore(Node, 2, AArch64::ST1Twov4s);
3157 SelectStore(Node, 2, AArch64::ST1Twov2d);
3160 SelectStore(Node, 2, AArch64::ST1Twov1d);
3165 case Intrinsic::aarch64_neon_st1x3: {
3167 SelectStore(Node, 3, AArch64::ST1Threev8b);
3170 SelectStore(Node, 3, AArch64::ST1Threev16b);
3173 SelectStore(Node, 3, AArch64::ST1Threev4h);
3176 SelectStore(Node, 3, AArch64::ST1Threev8h);
3179 SelectStore(Node, 3, AArch64::ST1Threev2s);
3182 SelectStore(Node, 3, AArch64::ST1Threev4s);
3185 SelectStore(Node, 3, AArch64::ST1Threev2d);
3188 SelectStore(Node, 3, AArch64::ST1Threev1d);
3193 case Intrinsic::aarch64_neon_st1x4: {
3195 SelectStore(Node, 4, AArch64::ST1Fourv8b);
3198 SelectStore(Node, 4, AArch64::ST1Fourv16b);
3201 SelectStore(Node, 4, AArch64::ST1Fourv4h);
3204 SelectStore(Node, 4, AArch64::ST1Fourv8h);
3207 SelectStore(Node, 4, AArch64::ST1Fourv2s);
3210 SelectStore(Node, 4, AArch64::ST1Fourv4s);
3213 SelectStore(Node, 4, AArch64::ST1Fourv2d);
3216 SelectStore(Node, 4, AArch64::ST1Fourv1d);
3221 case Intrinsic::aarch64_neon_st2: {
3223 SelectStore(Node, 2, AArch64::ST2Twov8b);
3226 SelectStore(Node, 2, AArch64::ST2Twov16b);
3229 SelectStore(Node, 2, AArch64::ST2Twov4h);
3232 SelectStore(Node, 2, AArch64::ST2Twov8h);
3235 SelectStore(Node, 2, AArch64::ST2Twov2s);
3238 SelectStore(Node, 2, AArch64::ST2Twov4s);
3241 SelectStore(Node, 2, AArch64::ST2Twov2d);
3244 SelectStore(Node, 2, AArch64::ST1Twov1d);
3249 case Intrinsic::aarch64_neon_st3: {
3251 SelectStore(Node, 3, AArch64::ST3Threev8b);
3254 SelectStore(Node, 3, AArch64::ST3Threev16b);
3257 SelectStore(Node, 3, AArch64::ST3Threev4h);
3260 SelectStore(Node, 3, AArch64::ST3Threev8h);
3263 SelectStore(Node, 3, AArch64::ST3Threev2s);
3266 SelectStore(Node, 3, AArch64::ST3Threev4s);
3269 SelectStore(Node, 3, AArch64::ST3Threev2d);
3272 SelectStore(Node, 3, AArch64::ST1Threev1d);
3277 case Intrinsic::aarch64_neon_st4: {
3279 SelectStore(Node, 4, AArch64::ST4Fourv8b);
3282 SelectStore(Node, 4, AArch64::ST4Fourv16b);
3285 SelectStore(Node, 4, AArch64::ST4Fourv4h);
3288 SelectStore(Node, 4, AArch64::ST4Fourv8h);
3291 SelectStore(Node, 4, AArch64::ST4Fourv2s);
3294 SelectStore(Node, 4, AArch64::ST4Fourv4s);
3297 SelectStore(Node, 4, AArch64::ST4Fourv2d);
3300 SelectStore(Node, 4, AArch64::ST1Fourv1d);
3305 case Intrinsic::aarch64_neon_st2lane: {
3307 SelectStoreLane(Node, 2, AArch64::ST2i8);
3311 SelectStoreLane(Node, 2, AArch64::ST2i16);
3315 SelectStoreLane(Node, 2, AArch64::ST2i32);
3319 SelectStoreLane(Node, 2, AArch64::ST2i64);
3324 case Intrinsic::aarch64_neon_st3lane: {
3326 SelectStoreLane(Node, 3, AArch64::ST3i8);
3330 SelectStoreLane(Node, 3, AArch64::ST3i16);
3334 SelectStoreLane(Node, 3, AArch64::ST3i32);
3338 SelectStoreLane(Node, 3, AArch64::ST3i64);
3343 case Intrinsic::aarch64_neon_st4lane: {
3345 SelectStoreLane(Node, 4, AArch64::ST4i8);
3349 SelectStoreLane(Node, 4, AArch64::ST4i16);
3353 SelectStoreLane(Node, 4, AArch64::ST4i32);
3357 SelectStoreLane(Node, 4, AArch64::ST4i64);
3367 SelectPostLoad(Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
3370 SelectPostLoad(Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
3373 SelectPostLoad(Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
3376 SelectPostLoad(Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
3379 SelectPostLoad(Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
3382 SelectPostLoad(Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
3385 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
3388 SelectPostLoad(Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
3395 SelectPostLoad(Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
3398 SelectPostLoad(Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
3401 SelectPostLoad(Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
3404 SelectPostLoad(Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
3407 SelectPostLoad(Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
3410 SelectPostLoad(Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
3413 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
3416 SelectPostLoad(Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
3423 SelectPostLoad(Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
3426 SelectPostLoad(Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
3429 SelectPostLoad(Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
3432 SelectPostLoad(Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
3435 SelectPostLoad(Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
3438 SelectPostLoad(Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
3441 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
3444 SelectPostLoad(Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
3451 SelectPostLoad(Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
3454 SelectPostLoad(Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
3457 SelectPostLoad(Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
3460 SelectPostLoad(Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
3463 SelectPostLoad(Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
3466 SelectPostLoad(Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
3469 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
3472 SelectPostLoad(Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
3479 SelectPostLoad(Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
3482 SelectPostLoad(Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
3485 SelectPostLoad(Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
3488 SelectPostLoad(Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
3491 SelectPostLoad(Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
3494 SelectPostLoad(Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
3497 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
3500 SelectPostLoad(Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
3507 SelectPostLoad(Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
3510 SelectPostLoad(Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
3513 SelectPostLoad(Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
3516 SelectPostLoad(Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
3519 SelectPostLoad(Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
3522 SelectPostLoad(Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
3525 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
3528 SelectPostLoad(Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
3535 SelectPostLoad(Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
3538 SelectPostLoad(Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
3541 SelectPostLoad(Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
3544 SelectPostLoad(Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
3547 SelectPostLoad(Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
3550 SelectPostLoad(Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
3553 SelectPostLoad(Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
3556 SelectPostLoad(Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
3563 SelectPostLoad(Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
3566 SelectPostLoad(Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
3569 SelectPostLoad(Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
3572 SelectPostLoad(Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
3575 SelectPostLoad(Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
3578 SelectPostLoad(Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
3581 SelectPostLoad(Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
3584 SelectPostLoad(Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
3591 SelectPostLoad(Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
3594 SelectPostLoad(Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
3597 SelectPostLoad(Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
3600 SelectPostLoad(Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
3603 SelectPostLoad(Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
3606 SelectPostLoad(Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
3609 SelectPostLoad(Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
3612 SelectPostLoad(Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
3619 SelectPostLoad(Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
3622 SelectPostLoad(Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
3625 SelectPostLoad(Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
3628 SelectPostLoad(Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
3631 SelectPostLoad(Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
3634 SelectPostLoad(Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
3637 SelectPostLoad(Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
3640 SelectPostLoad(Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
3647 SelectPostLoadLane(Node, 1, AArch64::LD1i8_POST);
3651 SelectPostLoadLane(Node, 1, AArch64::LD1i16_POST);
3655 SelectPostLoadLane(Node, 1, AArch64::LD1i32_POST);
3659 SelectPostLoadLane(Node, 1, AArch64::LD1i64_POST);
3666 SelectPostLoadLane(Node, 2, AArch64::LD2i8_POST);
3670 SelectPostLoadLane(Node, 2, AArch64::LD2i16_POST);
3674 SelectPostLoadLane(Node, 2, AArch64::LD2i32_POST);
3678 SelectPostLoadLane(Node, 2, AArch64::LD2i64_POST);
3685 SelectPostLoadLane(Node, 3, AArch64::LD3i8_POST);
3689 SelectPostLoadLane(Node, 3, AArch64::LD3i16_POST);
3693 SelectPostLoadLane(Node, 3, AArch64::LD3i32_POST);
3697 SelectPostLoadLane(Node, 3, AArch64::LD3i64_POST);
3704 SelectPostLoadLane(Node, 4, AArch64::LD4i8_POST);
3708 SelectPostLoadLane(Node, 4, AArch64::LD4i16_POST);
3712 SelectPostLoadLane(Node, 4, AArch64::LD4i32_POST);
3716 SelectPostLoadLane(Node, 4, AArch64::LD4i64_POST);
3724 SelectPostStore(Node, 2, AArch64::ST2Twov8b_POST);
3727 SelectPostStore(Node, 2, AArch64::ST2Twov16b_POST);
3730 SelectPostStore(Node, 2, AArch64::ST2Twov4h_POST);
3733 SelectPostStore(Node, 2, AArch64::ST2Twov8h_POST);
3736 SelectPostStore(Node, 2, AArch64::ST2Twov2s_POST);
3739 SelectPostStore(Node, 2, AArch64::ST2Twov4s_POST);
3742 SelectPostStore(Node, 2, AArch64::ST2Twov2d_POST);
3745 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
3753 SelectPostStore(Node, 3, AArch64::ST3Threev8b_POST);
3756 SelectPostStore(Node, 3, AArch64::ST3Threev16b_POST);
3759 SelectPostStore(Node, 3, AArch64::ST3Threev4h_POST);
3762 SelectPostStore(Node, 3, AArch64::ST3Threev8h_POST);
3765 SelectPostStore(Node, 3, AArch64::ST3Threev2s_POST);
3768 SelectPostStore(Node, 3, AArch64::ST3Threev4s_POST);
3771 SelectPostStore(Node, 3, AArch64::ST3Threev2d_POST);
3774 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
3782 SelectPostStore(Node, 4, AArch64::ST4Fourv8b_POST);
3785 SelectPostStore(Node, 4, AArch64::ST4Fourv16b_POST);
3788 SelectPostStore(Node, 4, AArch64::ST4Fourv4h_POST);
3791 SelectPostStore(Node, 4, AArch64::ST4Fourv8h_POST);
3794 SelectPostStore(Node, 4, AArch64::ST4Fourv2s_POST);
3797 SelectPostStore(Node, 4, AArch64::ST4Fourv4s_POST);
3800 SelectPostStore(Node, 4, AArch64::ST4Fourv2d_POST);
3803 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
3811 SelectPostStore(Node, 2, AArch64::ST1Twov8b_POST);
3814 SelectPostStore(Node, 2, AArch64::ST1Twov16b_POST);
3817 SelectPostStore(Node, 2, AArch64::ST1Twov4h_POST);
3820 SelectPostStore(Node, 2, AArch64::ST1Twov8h_POST);
3823 SelectPostStore(Node, 2, AArch64::ST1Twov2s_POST);
3826 SelectPostStore(Node, 2, AArch64::ST1Twov4s_POST);
3829 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
3832 SelectPostStore(Node, 2, AArch64::ST1Twov2d_POST);
3840 SelectPostStore(Node, 3, AArch64::ST1Threev8b_POST);
3843 SelectPostStore(Node, 3, AArch64::ST1Threev16b_POST);
3846 SelectPostStore(Node, 3, AArch64::ST1Threev4h_POST);
3849 SelectPostStore(Node, 3, AArch64::ST1Threev8h_POST);
3852 SelectPostStore(Node, 3, AArch64::ST1Threev2s_POST);
3855 SelectPostStore(Node, 3, AArch64::ST1Threev4s_POST);
3858 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
3861 SelectPostStore(Node, 3, AArch64::ST1Threev2d_POST);
3869 SelectPostStore(Node, 4, AArch64::ST1Fourv8b_POST);
3872 SelectPostStore(Node, 4, AArch64::ST1Fourv16b_POST);
3875 SelectPostStore(Node, 4, AArch64::ST1Fourv4h_POST);
3878 SelectPostStore(Node, 4, AArch64::ST1Fourv8h_POST);
3881 SelectPostStore(Node, 4, AArch64::ST1Fourv2s_POST);
3884 SelectPostStore(Node, 4, AArch64::ST1Fourv4s_POST);
3887 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
3890 SelectPostStore(Node, 4, AArch64::ST1Fourv2d_POST);
3898 SelectPostStoreLane(Node, 2, AArch64::ST2i8_POST);
3902 SelectPostStoreLane(Node, 2, AArch64::ST2i16_POST);
3906 SelectPostStoreLane(Node, 2, AArch64::ST2i32_POST);
3910 SelectPostStoreLane(Node, 2, AArch64::ST2i64_POST);
3918 SelectPostStoreLane(Node, 3, AArch64::ST3i8_POST);
3922 SelectPostStoreLane(Node, 3, AArch64::ST3i16_POST);
3926 SelectPostStoreLane(Node, 3, AArch64::ST3i32_POST);
3930 SelectPostStoreLane(Node, 3, AArch64::ST3i64_POST);
3938 SelectPostStoreLane(Node, 4, AArch64::ST4i8_POST);
3942 SelectPostStoreLane(Node, 4, AArch64::ST4i16_POST);
3946 SelectPostStoreLane(Node, 4, AArch64::ST4i32_POST);
3950 SelectPostStoreLane(Node, 4, AArch64::ST4i64_POST);
3965 return new AArch64DAGToDAGISel(TM, OptLevel);
void clearAllBits()
Set every bit to 0.
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &ShiftAmount, int &MaskWidth)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL...
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void flipAllBits()
Toggle every bit to its opposite value.
uint64_t getZExtValue() const
Get zero extended value.
DiagnosticInfoOptimizationBase::Argument NV
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
Type * getValueType() const
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
const GlobalValue * getGlobal() const
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
void setNodeId(int Id)
Set unique node id.
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
bool optForSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
void dumpr() const
Dump (recursively) this node and its use-def subgraph.
A description of a memory reference used in the backend.
static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp, SDValue &LaneOp, int &LaneIdx)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static bool isPreferredADD(int64_t ImmOff)
Shift and rotation operations.
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Reg
All possible values of the reg field in the ModR/M byte.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
APInt shl(unsigned shiftAmt) const
Left-shift function.
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
constexpr bool isMask_64(uint64_t Value)
isMask_64 - This function returns true if the argument is a non-empty sequence of ones starting at th...
Simple integer binary arithmetic operators.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
size_t size() const
size - Get the array size.
static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG)
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 regis...
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
SDNode * getNode() const
get the SDNode which holds the desired result
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
const Constant * getConstVal() const
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
unsigned getScalarValueSizeInBits() const
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32...
unsigned countPopulation() const
Count the number of bits set.
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
EVT - Extended Value Type.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
const SDValue & getOffset() const
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
const MDOperand & getOperand(unsigned I) const
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
void dump() const
Dump this node, for debugging.
static bool isStrongerThanMonotonic(AtomicOrdering ao)
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDNode * SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type...
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
iterator_range< use_iterator > uses()
int64_t getSExtValue() const
op_iterator op_begin() const
const SysReg * lookupSysRegByName(StringRef)
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
uint32_t parseGenericRegister(StringRef Name)
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOpt::Level OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG...
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
constexpr bool isShiftedMask_64(uint64_t Value)
isShiftedMask_64 - This function returns true if the argument contains a non-empty sequence of ones w...
static bool checkHighLaneIndex(SDNode *DL, SDValue &LaneOp, int &LaneIdx)
Bitwise operators - logical and, logical or, logical xor.
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static int getIntOperandFromRegisterString(StringRef RegString)
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted, suitable for use in a BFI instruction.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
EVT getValueType() const
Return the ValueType of the referenced return value.
OutputIt transform(R &&Range, OutputIt d_first, UnaryPredicate P)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere...
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool is128BitVector() const
is128BitVector - Return true if this is a 128-bit vector type.
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
const MDNode * getMD() const
unsigned getAlignment() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
constexpr bool isShiftedMask_32(uint32_t Value)
isShiftedMask_32 - This function returns true if the argument contains a non-empty sequence of ones w...
StringRef - Represent a constant reference to a string, i.e.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
TRUNCATE - Completely drop the high bits.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
bool is64BitVector() const
is64BitVector - Return true if this is a 64-bit vector type.
std::size_t countLeadingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the most significant bit to the first zero bit.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
uint64_t getZExtValue() const
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.