38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
87 !Subtarget.hasStdExtF()) {
88 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
89 "doesn't support the F instruction set extension (ignoring "
93 !Subtarget.hasStdExtD()) {
94 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
95 "doesn't support the D instruction set extension (ignoring "
119 if (Subtarget.hasStdExtZfhmin())
121 if (Subtarget.hasStdExtZfbfmin())
123 if (Subtarget.hasStdExtF())
125 if (Subtarget.hasStdExtD())
127 if (Subtarget.hasStdExtZhinxmin())
129 if (Subtarget.hasStdExtZfinx())
131 if (Subtarget.hasStdExtZdinx()) {
139 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
140 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
142 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
143 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
144 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
145 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
146 MVT::nxv4i64, MVT::nxv8i64};
148 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
149 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
151 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
152 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
154 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
156 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
159 auto addRegClassForRVV = [
this](
MVT VT) {
163 if (VT.getVectorMinNumElements() < MinElts)
166 unsigned Size = VT.getSizeInBits().getKnownMinValue();
169 RC = &RISCV::VRRegClass;
171 RC = &RISCV::VRM2RegClass;
173 RC = &RISCV::VRM4RegClass;
175 RC = &RISCV::VRM8RegClass;
182 for (
MVT VT : BoolVecVTs)
183 addRegClassForRVV(VT);
184 for (
MVT VT : IntVecVTs) {
185 if (VT.getVectorElementType() == MVT::i64 &&
188 addRegClassForRVV(VT);
192 for (
MVT VT : F16VecVTs)
193 addRegClassForRVV(VT);
196 for (
MVT VT : BF16VecVTs)
197 addRegClassForRVV(VT);
200 for (
MVT VT : F32VecVTs)
201 addRegClassForRVV(VT);
204 for (
MVT VT : F64VecVTs)
205 addRegClassForRVV(VT);
208 auto addRegClassForFixedVectors = [
this](
MVT VT) {
215 if (useRVVForFixedLengthVectorVT(VT))
216 addRegClassForFixedVectors(VT);
219 if (useRVVForFixedLengthVectorVT(VT))
220 addRegClassForFixedVectors(VT);
247 if (!(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
261 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
262 !(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()))
272 if (!Subtarget.hasStdExtZbb())
278 if (!Subtarget.hasStdExtZmmul()) {
280 }
else if (Subtarget.
is64Bit()) {
287 if (!Subtarget.hasStdExtM()) {
290 }
else if (Subtarget.
is64Bit()) {
292 {MVT::i8, MVT::i16, MVT::i32},
Custom);
302 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
305 }
else if (Subtarget.hasVendorXTHeadBb()) {
309 }
else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
318 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
319 Subtarget.hasVendorXTHeadBb())
323 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
331 if (Subtarget.hasStdExtZbb() ||
332 (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
337 if (Subtarget.hasStdExtZbb() ||
338 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
345 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
346 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
355 if (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()) {
357 }
else if (Subtarget.hasShortForwardBranchOpt()) {
360 }
else if (Subtarget.
is64Bit()) {
364 if (!Subtarget.hasVendorXTHeadCondMov())
367 static const unsigned FPLegalNodeTypes[] = {
380 static const unsigned FPOpToExpand[] = {
384 static const unsigned FPRndMode[] = {
391 static const unsigned ZfhminZfbfminPromoteOps[] = {
402 if (Subtarget.hasStdExtZfbfmin()) {
485 if (Subtarget.hasStdExtZfa()) {
502 if (Subtarget.hasStdExtZfa()) {
574 if (Subtarget.hasStdExtZicbop()) {
578 if (Subtarget.hasStdExtA()) {
580 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
584 }
else if (Subtarget.hasForcedAtomics()) {
608 {MVT::i8, MVT::i16},
Custom);
619 static const unsigned IntegerVPOps[] = {
620 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
621 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
622 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
623 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
624 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
625 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
626 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
627 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
628 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
629 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
630 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
631 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
632 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
633 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
634 ISD::EXPERIMENTAL_VP_SPLAT};
636 static const unsigned FloatingPointVPOps[] = {
637 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
638 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
639 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
640 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
641 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
642 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
643 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
644 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
645 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
646 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
647 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
648 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
649 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
650 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
652 static const unsigned IntegerVecReduceOps[] = {
657 static const unsigned FloatingPointVecReduceOps[] = {
670 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
671 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
672 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
676 for (
MVT VT : BoolVecVTs) {
706 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
730 ISD::VP_TRUNCATE, ISD::VP_SETCC},
746 for (
MVT VT : IntVecVTs) {
757 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
805 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
806 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
830 if (Subtarget.hasStdExtZvkb()) {
838 if (Subtarget.hasStdExtZvbb()) {
842 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
848 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
857 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
877 static const unsigned ZvfhminPromoteOps[] = {
887 static const unsigned ZvfhminPromoteVPOps[] = {
888 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
889 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
890 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
891 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
892 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
893 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
894 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
895 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
896 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
899 const auto SetCommonVFPActions = [&](
MVT VT) {
944 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
945 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
976 const auto SetCommonVFPExtLoadTruncStoreActions =
978 for (
auto SmallVT : SmallerVTs) {
985 for (
MVT VT : F16VecVTs) {
988 SetCommonVFPActions(VT);
991 for (
MVT VT : F16VecVTs) {
1002 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1007 if (Subtarget.hasStdExtZfhmin())
1013 if (VT == MVT::nxv32f16) {
1027 for (
MVT VT : BF16VecVTs) {
1038 if (Subtarget.hasStdExtZfbfmin())
1048 for (
MVT VT : F32VecVTs) {
1051 SetCommonVFPActions(VT);
1052 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1057 for (
MVT VT : F64VecVTs) {
1060 SetCommonVFPActions(VT);
1061 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1062 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1068 if (!useRVVForFixedLengthVectorVT(VT))
1111 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1138 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1160 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1161 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1198 if (Subtarget.hasStdExtZvkb())
1201 if (Subtarget.hasStdExtZvbb()) {
1223 if (!useRVVForFixedLengthVectorVT(VT))
1253 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1256 if (Subtarget.hasStdExtZfhmin()) {
1294 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1295 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1341 if (Subtarget.hasStdExtA())
1344 if (Subtarget.hasForcedAtomics()) {
1354 if (Subtarget.hasVendorXTHeadMemIdx()) {
1370 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
1395 if (Subtarget.hasStdExtZbb())
1398 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1402 if (Subtarget.hasStdExtZbkb())
1414 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1417 if (Subtarget.hasVendorXTHeadMemPair())
1445MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1450bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1452 bool IsScalable)
const {
1459 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1484 unsigned Intrinsic)
const {
1485 auto &
DL =
I.getDataLayout();
1487 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1488 bool IsUnitStrided,
bool UsePtrVal =
false) {
1493 Info.ptrVal =
I.getArgOperand(PtrOp);
1495 Info.fallbackAddressSpace =
1496 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1500 MemTy =
I.getArgOperand(0)->getType();
1503 MemTy =
I.getType();
1518 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1522 switch (Intrinsic) {
1525 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1526 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1527 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1528 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1529 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1530 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1531 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1532 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1533 case Intrinsic::riscv_masked_cmpxchg_i32:
1535 Info.memVT = MVT::i32;
1536 Info.ptrVal =
I.getArgOperand(0);
1542 case Intrinsic::riscv_seg2_load:
1543 case Intrinsic::riscv_seg3_load:
1544 case Intrinsic::riscv_seg4_load:
1545 case Intrinsic::riscv_seg5_load:
1546 case Intrinsic::riscv_seg6_load:
1547 case Intrinsic::riscv_seg7_load:
1548 case Intrinsic::riscv_seg8_load:
1549 return SetRVVLoadStoreInfo( 0,
false,
1551 case Intrinsic::riscv_seg2_store:
1552 case Intrinsic::riscv_seg3_store:
1553 case Intrinsic::riscv_seg4_store:
1554 case Intrinsic::riscv_seg5_store:
1555 case Intrinsic::riscv_seg6_store:
1556 case Intrinsic::riscv_seg7_store:
1557 case Intrinsic::riscv_seg8_store:
1559 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1562 case Intrinsic::riscv_vle:
1563 case Intrinsic::riscv_vle_mask:
1564 case Intrinsic::riscv_vleff:
1565 case Intrinsic::riscv_vleff_mask:
1566 return SetRVVLoadStoreInfo( 1,
1570 case Intrinsic::riscv_vse:
1571 case Intrinsic::riscv_vse_mask:
1572 return SetRVVLoadStoreInfo( 1,
1576 case Intrinsic::riscv_vlse:
1577 case Intrinsic::riscv_vlse_mask:
1578 case Intrinsic::riscv_vloxei:
1579 case Intrinsic::riscv_vloxei_mask:
1580 case Intrinsic::riscv_vluxei:
1581 case Intrinsic::riscv_vluxei_mask:
1582 return SetRVVLoadStoreInfo( 1,
1585 case Intrinsic::riscv_vsse:
1586 case Intrinsic::riscv_vsse_mask:
1587 case Intrinsic::riscv_vsoxei:
1588 case Intrinsic::riscv_vsoxei_mask:
1589 case Intrinsic::riscv_vsuxei:
1590 case Intrinsic::riscv_vsuxei_mask:
1591 return SetRVVLoadStoreInfo( 1,
1594 case Intrinsic::riscv_vlseg2:
1595 case Intrinsic::riscv_vlseg3:
1596 case Intrinsic::riscv_vlseg4:
1597 case Intrinsic::riscv_vlseg5:
1598 case Intrinsic::riscv_vlseg6:
1599 case Intrinsic::riscv_vlseg7:
1600 case Intrinsic::riscv_vlseg8:
1601 case Intrinsic::riscv_vlseg2ff:
1602 case Intrinsic::riscv_vlseg3ff:
1603 case Intrinsic::riscv_vlseg4ff:
1604 case Intrinsic::riscv_vlseg5ff:
1605 case Intrinsic::riscv_vlseg6ff:
1606 case Intrinsic::riscv_vlseg7ff:
1607 case Intrinsic::riscv_vlseg8ff:
1608 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1611 case Intrinsic::riscv_vlseg2_mask:
1612 case Intrinsic::riscv_vlseg3_mask:
1613 case Intrinsic::riscv_vlseg4_mask:
1614 case Intrinsic::riscv_vlseg5_mask:
1615 case Intrinsic::riscv_vlseg6_mask:
1616 case Intrinsic::riscv_vlseg7_mask:
1617 case Intrinsic::riscv_vlseg8_mask:
1618 case Intrinsic::riscv_vlseg2ff_mask:
1619 case Intrinsic::riscv_vlseg3ff_mask:
1620 case Intrinsic::riscv_vlseg4ff_mask:
1621 case Intrinsic::riscv_vlseg5ff_mask:
1622 case Intrinsic::riscv_vlseg6ff_mask:
1623 case Intrinsic::riscv_vlseg7ff_mask:
1624 case Intrinsic::riscv_vlseg8ff_mask:
1625 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1628 case Intrinsic::riscv_vlsseg2:
1629 case Intrinsic::riscv_vlsseg3:
1630 case Intrinsic::riscv_vlsseg4:
1631 case Intrinsic::riscv_vlsseg5:
1632 case Intrinsic::riscv_vlsseg6:
1633 case Intrinsic::riscv_vlsseg7:
1634 case Intrinsic::riscv_vlsseg8:
1635 case Intrinsic::riscv_vloxseg2:
1636 case Intrinsic::riscv_vloxseg3:
1637 case Intrinsic::riscv_vloxseg4:
1638 case Intrinsic::riscv_vloxseg5:
1639 case Intrinsic::riscv_vloxseg6:
1640 case Intrinsic::riscv_vloxseg7:
1641 case Intrinsic::riscv_vloxseg8:
1642 case Intrinsic::riscv_vluxseg2:
1643 case Intrinsic::riscv_vluxseg3:
1644 case Intrinsic::riscv_vluxseg4:
1645 case Intrinsic::riscv_vluxseg5:
1646 case Intrinsic::riscv_vluxseg6:
1647 case Intrinsic::riscv_vluxseg7:
1648 case Intrinsic::riscv_vluxseg8:
1649 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1652 case Intrinsic::riscv_vlsseg2_mask:
1653 case Intrinsic::riscv_vlsseg3_mask:
1654 case Intrinsic::riscv_vlsseg4_mask:
1655 case Intrinsic::riscv_vlsseg5_mask:
1656 case Intrinsic::riscv_vlsseg6_mask:
1657 case Intrinsic::riscv_vlsseg7_mask:
1658 case Intrinsic::riscv_vlsseg8_mask:
1659 case Intrinsic::riscv_vloxseg2_mask:
1660 case Intrinsic::riscv_vloxseg3_mask:
1661 case Intrinsic::riscv_vloxseg4_mask:
1662 case Intrinsic::riscv_vloxseg5_mask:
1663 case Intrinsic::riscv_vloxseg6_mask:
1664 case Intrinsic::riscv_vloxseg7_mask:
1665 case Intrinsic::riscv_vloxseg8_mask:
1666 case Intrinsic::riscv_vluxseg2_mask:
1667 case Intrinsic::riscv_vluxseg3_mask:
1668 case Intrinsic::riscv_vluxseg4_mask:
1669 case Intrinsic::riscv_vluxseg5_mask:
1670 case Intrinsic::riscv_vluxseg6_mask:
1671 case Intrinsic::riscv_vluxseg7_mask:
1672 case Intrinsic::riscv_vluxseg8_mask:
1673 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1676 case Intrinsic::riscv_vsseg2:
1677 case Intrinsic::riscv_vsseg3:
1678 case Intrinsic::riscv_vsseg4:
1679 case Intrinsic::riscv_vsseg5:
1680 case Intrinsic::riscv_vsseg6:
1681 case Intrinsic::riscv_vsseg7:
1682 case Intrinsic::riscv_vsseg8:
1683 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1686 case Intrinsic::riscv_vsseg2_mask:
1687 case Intrinsic::riscv_vsseg3_mask:
1688 case Intrinsic::riscv_vsseg4_mask:
1689 case Intrinsic::riscv_vsseg5_mask:
1690 case Intrinsic::riscv_vsseg6_mask:
1691 case Intrinsic::riscv_vsseg7_mask:
1692 case Intrinsic::riscv_vsseg8_mask:
1693 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1696 case Intrinsic::riscv_vssseg2:
1697 case Intrinsic::riscv_vssseg3:
1698 case Intrinsic::riscv_vssseg4:
1699 case Intrinsic::riscv_vssseg5:
1700 case Intrinsic::riscv_vssseg6:
1701 case Intrinsic::riscv_vssseg7:
1702 case Intrinsic::riscv_vssseg8:
1703 case Intrinsic::riscv_vsoxseg2:
1704 case Intrinsic::riscv_vsoxseg3:
1705 case Intrinsic::riscv_vsoxseg4:
1706 case Intrinsic::riscv_vsoxseg5:
1707 case Intrinsic::riscv_vsoxseg6:
1708 case Intrinsic::riscv_vsoxseg7:
1709 case Intrinsic::riscv_vsoxseg8:
1710 case Intrinsic::riscv_vsuxseg2:
1711 case Intrinsic::riscv_vsuxseg3:
1712 case Intrinsic::riscv_vsuxseg4:
1713 case Intrinsic::riscv_vsuxseg5:
1714 case Intrinsic::riscv_vsuxseg6:
1715 case Intrinsic::riscv_vsuxseg7:
1716 case Intrinsic::riscv_vsuxseg8:
1717 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1720 case Intrinsic::riscv_vssseg2_mask:
1721 case Intrinsic::riscv_vssseg3_mask:
1722 case Intrinsic::riscv_vssseg4_mask:
1723 case Intrinsic::riscv_vssseg5_mask:
1724 case Intrinsic::riscv_vssseg6_mask:
1725 case Intrinsic::riscv_vssseg7_mask:
1726 case Intrinsic::riscv_vssseg8_mask:
1727 case Intrinsic::riscv_vsoxseg2_mask:
1728 case Intrinsic::riscv_vsoxseg3_mask:
1729 case Intrinsic::riscv_vsoxseg4_mask:
1730 case Intrinsic::riscv_vsoxseg5_mask:
1731 case Intrinsic::riscv_vsoxseg6_mask:
1732 case Intrinsic::riscv_vsoxseg7_mask:
1733 case Intrinsic::riscv_vsoxseg8_mask:
1734 case Intrinsic::riscv_vsuxseg2_mask:
1735 case Intrinsic::riscv_vsuxseg3_mask:
1736 case Intrinsic::riscv_vsuxseg4_mask:
1737 case Intrinsic::riscv_vsuxseg5_mask:
1738 case Intrinsic::riscv_vsuxseg6_mask:
1739 case Intrinsic::riscv_vsuxseg7_mask:
1740 case Intrinsic::riscv_vsuxseg8_mask:
1741 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1782 return isInt<12>(Imm);
1786 return isInt<12>(Imm);
1799 return (SrcBits == 64 && DestBits == 32);
1810 return (SrcBits == 64 && DestBits == 32);
1821 if (SrcBits == DestBits * 2) {
1832 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1833 EVT MemVT = LD->getMemoryVT();
1834 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1844 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1852 return Subtarget.hasStdExtZbb() ||
1853 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1857 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1858 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1869 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1874 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1878 EVT VT =
Y.getValueType();
1884 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1885 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
1890 if (Subtarget.hasStdExtZbs())
1891 return X.getValueType().isScalarInteger();
1892 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1894 if (Subtarget.hasVendorXTHeadBs())
1895 return C !=
nullptr;
1897 return C &&
C->getAPIntValue().ule(10);
1917 if (BitSize > Subtarget.
getXLen())
1921 int64_t Val = Imm.getSExtValue();
1929 if (!Subtarget.enableUnalignedScalarMem())
1945 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1952 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1956 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1968 case Instruction::Add:
1969 case Instruction::Sub:
1970 case Instruction::Mul:
1971 case Instruction::And:
1972 case Instruction::Or:
1973 case Instruction::Xor:
1974 case Instruction::FAdd:
1975 case Instruction::FSub:
1976 case Instruction::FMul:
1977 case Instruction::FDiv:
1978 case Instruction::ICmp:
1979 case Instruction::FCmp:
1981 case Instruction::Shl:
1982 case Instruction::LShr:
1983 case Instruction::AShr:
1984 case Instruction::UDiv:
1985 case Instruction::SDiv:
1986 case Instruction::URem:
1987 case Instruction::SRem:
1988 case Instruction::Select:
1989 return Operand == 1;
2003 auto *
II = dyn_cast<IntrinsicInst>(
I);
2007 switch (
II->getIntrinsicID()) {
2008 case Intrinsic::fma:
2009 case Intrinsic::vp_fma:
2010 return Operand == 0 || Operand == 1;
2011 case Intrinsic::vp_shl:
2012 case Intrinsic::vp_lshr:
2013 case Intrinsic::vp_ashr:
2014 case Intrinsic::vp_udiv:
2015 case Intrinsic::vp_sdiv:
2016 case Intrinsic::vp_urem:
2017 case Intrinsic::vp_srem:
2018 case Intrinsic::ssub_sat:
2019 case Intrinsic::vp_ssub_sat:
2020 case Intrinsic::usub_sat:
2021 case Intrinsic::vp_usub_sat:
2022 return Operand == 1;
2024 case Intrinsic::vp_add:
2025 case Intrinsic::vp_mul:
2026 case Intrinsic::vp_and:
2027 case Intrinsic::vp_or:
2028 case Intrinsic::vp_xor:
2029 case Intrinsic::vp_fadd:
2030 case Intrinsic::vp_fmul:
2031 case Intrinsic::vp_icmp:
2032 case Intrinsic::vp_fcmp:
2033 case Intrinsic::smin:
2034 case Intrinsic::vp_smin:
2035 case Intrinsic::umin:
2036 case Intrinsic::vp_umin:
2037 case Intrinsic::smax:
2038 case Intrinsic::vp_smax:
2039 case Intrinsic::umax:
2040 case Intrinsic::vp_umax:
2041 case Intrinsic::sadd_sat:
2042 case Intrinsic::vp_sadd_sat:
2043 case Intrinsic::uadd_sat:
2044 case Intrinsic::vp_uadd_sat:
2046 case Intrinsic::vp_sub:
2047 case Intrinsic::vp_fsub:
2048 case Intrinsic::vp_fdiv:
2049 return Operand == 0 || Operand == 1;
2070 if (!Subtarget.sinkSplatOperands())
2073 for (
auto OpIdx :
enumerate(
I->operands())) {
2077 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2079 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2088 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2093 for (
Use &U :
Op->uses()) {
2145 if (!Subtarget.hasStdExtZfa())
2146 return std::make_pair(-1,
false);
2148 bool IsSupportedVT =
false;
2149 if (VT == MVT::f16) {
2150 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2151 }
else if (VT == MVT::f32) {
2152 IsSupportedVT =
true;
2153 }
else if (VT == MVT::f64) {
2154 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2155 IsSupportedVT =
true;
2159 return std::make_pair(-1,
false);
2162 if (
Index < 0 && Imm.isNegative())
2166 return std::make_pair(
Index,
false);
2170 bool ForCodeSize)
const {
2171 bool IsLegalVT =
false;
2174 else if (VT == MVT::f32)
2176 else if (VT == MVT::f64)
2178 else if (VT == MVT::bf16)
2179 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2191 return Imm.isZero();
2195 if (Imm.isNegZero())
2208 unsigned Index)
const {
2221 if (EltVT == MVT::i1)
2234 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2241 if ((ResElts * 2) != SrcElts)
2282 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2284 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2299 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2305 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2318 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2319 int64_t
C = RHSC->getSExtValue();
2361 switch (KnownSize) {
2389 return RISCV::VRRegClassID;
2391 return RISCV::VRM2RegClassID;
2393 return RISCV::VRM4RegClassID;
2395 return RISCV::VRM8RegClassID;
2405 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2406 "Unexpected subreg numbering");
2407 return RISCV::sub_vrm1_0 +
Index;
2410 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2411 "Unexpected subreg numbering");
2412 return RISCV::sub_vrm2_0 +
Index;
2415 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2416 "Unexpected subreg numbering");
2417 return RISCV::sub_vrm4_0 +
Index;
2424 return RISCV::VRRegClassID;
2433std::pair<unsigned, unsigned>
2435 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2437 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2438 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2439 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2440 "Register classes not ordered");
2449 unsigned SubRegIdx = RISCV::NoSubRegister;
2450 for (
const unsigned RCID :
2451 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2452 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2456 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2461 return {SubRegIdx, InsertExtractIdx};
2466bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2495unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2502 "Unexpected opcode");
2504 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2506 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2509 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2583bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2584 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2593 "Expected legal fixed length vector!");
2596 unsigned MaxELen = Subtarget.
getELen();
2630 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2637 "Expected to convert into a scalable vector!");
2638 assert(V.getValueType().isFixedLengthVector() &&
2639 "Expected a fixed length vector operand!");
2649 "Expected to convert into a fixed length vector!");
2650 assert(V.getValueType().isScalableVector() &&
2651 "Expected a scalable vector operand!");
2674static std::pair<SDValue, SDValue>
2683static std::pair<SDValue, SDValue>
2696static std::pair<SDValue, SDValue>
2713std::pair<unsigned, unsigned>
2729 return std::make_pair(MinVLMAX, MaxVLMAX);
2741 EVT VT,
unsigned DefinedValues)
const {
2755 std::tie(LMul, Fractional) =
2758 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2760 Cost = (LMul * DLenFactor);
2805 MVT DstVT =
Op.getSimpleValueType();
2806 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2814 Src.getValueType() == MVT::bf16) {
2821 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2829 Opc,
DL, DstVT, Src,
2843 MVT SrcVT = Src.getSimpleValueType();
2849 if (SatVT != DstEltVT)
2852 MVT DstContainerVT = DstVT;
2853 MVT SrcContainerVT = SrcVT;
2859 "Expected same element count");
2868 {Src, Src, DAG.getCondCode(ISD::SETNE),
2869 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2873 if (DstEltSize > (2 * SrcEltSize)) {
2879 MVT CvtContainerVT = DstContainerVT;
2880 MVT CvtEltVT = DstEltVT;
2881 if (SrcEltSize > (2 * DstEltSize)) {
2890 while (CvtContainerVT != DstContainerVT) {
2896 Res = DAG.
getNode(ClipOpc,
DL, CvtContainerVT, Res, Mask, VL);
2903 Res, DAG.
getUNDEF(DstContainerVT), VL);
2915 case ISD::VP_FROUNDEVEN:
2919 case ISD::VP_FROUNDTOZERO:
2923 case ISD::VP_FFLOOR:
2931 case ISD::VP_FROUND:
2947 MVT VT =
Op.getSimpleValueType();
2954 MVT ContainerVT = VT;
2961 if (
Op->isVPOpcode()) {
2962 Mask =
Op.getOperand(1);
2966 VL =
Op.getOperand(2);
2988 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3002 switch (
Op.getOpcode()) {
3008 case ISD::VP_FFLOOR:
3011 case ISD::VP_FROUND:
3012 case ISD::VP_FROUNDEVEN:
3013 case ISD::VP_FROUNDTOZERO: {
3029 case ISD::VP_FNEARBYINT:
3042 Src, Src, Mask, VL);
3057 MVT VT =
Op.getSimpleValueType();
3061 MVT ContainerVT = VT;
3073 MVT MaskVT = Mask.getSimpleValueType();
3076 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3077 DAG.getUNDEF(MaskVT), Mask, VL});
3081 {Chain, Src, Src, Src, Unorder, VL});
3082 Chain = Src.getValue(1);
3098 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3110 switch (
Op.getOpcode()) {
3121 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3127 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3131 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3140 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3141 Truncated, Mask, VL);
3147 Src, Src, Mask, VL);
3157 MVT VT =
Op.getSimpleValueType();
3185 MVT VT =
Op.getSimpleValueType();
3190 MVT ContainerVT = VT;
3233 "Unexpected vector MVT");
3261 return std::nullopt;
3279 unsigned EltSizeInBits) {
3282 return std::nullopt;
3283 bool IsInteger =
Op.getValueType().isInteger();
3285 std::optional<unsigned> SeqStepDenom;
3286 std::optional<APInt> SeqStepNum;
3287 std::optional<APInt> SeqAddend;
3288 std::optional<std::pair<APInt, unsigned>> PrevElt;
3289 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3294 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3296 if (Elt.isUndef()) {
3297 Elts[
Idx] = std::nullopt;
3301 Elts[
Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
3306 return std::nullopt;
3307 Elts[
Idx] = *ExactInteger;
3320 unsigned IdxDiff =
Idx - PrevElt->second;
3321 APInt ValDiff = *Elt - PrevElt->first;
3329 int64_t Remainder = ValDiff.
srem(IdxDiff);
3334 return std::nullopt;
3335 ValDiff = ValDiff.
sdiv(IdxDiff);
3340 SeqStepNum = ValDiff;
3341 else if (ValDiff != SeqStepNum)
3342 return std::nullopt;
3345 SeqStepDenom = IdxDiff;
3346 else if (IdxDiff != *SeqStepDenom)
3347 return std::nullopt;
3351 if (!PrevElt || PrevElt->first != *Elt)
3352 PrevElt = std::make_pair(*Elt,
Idx);
3356 if (!SeqStepNum || !SeqStepDenom)
3357 return std::nullopt;
3365 (
APInt(EltSizeInBits,
Idx) * *SeqStepNum).sdiv(*SeqStepDenom);
3367 APInt Addend = *Elt - ExpectedVal;
3370 else if (Addend != SeqAddend)
3371 return std::nullopt;
3374 assert(SeqAddend &&
"Must have an addend if we have a step");
3376 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
3377 SeqAddend->getSExtValue()};
3398 MVT ContainerVT = VT;
3426 MVT VT =
Op.getSimpleValueType();
3438 unsigned MostCommonCount = 0;
3440 unsigned NumUndefElts =
3448 unsigned NumScalarLoads = 0;
3454 ValueCounts.
insert(std::make_pair(V, 0));
3455 unsigned &Count = ValueCounts[V];
3457 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3458 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3463 if (++Count >= MostCommonCount) {
3465 MostCommonCount = Count;
3469 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3470 unsigned NumDefElts = NumElts - NumUndefElts;
3471 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3477 ((MostCommonCount > DominantValueCountThreshold) ||
3490 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3491 LastOp != DominantValue) {
3500 Processed.insert(LastOp);
3505 const SDValue &V = OpIdx.value();
3506 if (V.isUndef() || !Processed.insert(V).second)
3508 if (ValueCounts[V] == 1) {
3517 return DAG.getConstant(V == V1, DL, XLenVT);
3533 MVT VT =
Op.getSimpleValueType();
3563 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3564 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3572 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3573 MVT IntegerViaVecVT =
3578 unsigned BitPos = 0, IntegerEltIdx = 0;
3581 for (
unsigned I = 0;
I < NumElts;) {
3583 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3584 Bits |= ((
uint64_t)BitValue << BitPos);
3590 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3591 if (NumViaIntegerBits <= 32)
3592 Bits = SignExtend64<32>(Bits);
3594 Elts[IntegerEltIdx] = Elt;
3603 if (NumElts < NumViaIntegerBits) {
3607 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3635 int64_t StepNumerator = SimpleVID->StepNumerator;
3636 unsigned StepDenominator = SimpleVID->StepDenominator;
3637 int64_t Addend = SimpleVID->Addend;
3639 assert(StepNumerator != 0 &&
"Invalid step");
3640 bool Negate =
false;
3641 int64_t SplatStepVal = StepNumerator;
3645 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3647 Negate = StepNumerator < 0;
3649 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3656 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3657 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3659 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3662 MVT VIDContainerVT =
3670 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3671 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3673 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3675 if (StepDenominator != 1) {
3680 if (Addend != 0 || Negate) {
3699 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3700 "Unexpected sequence type");
3704 unsigned ViaVecLen =
3708 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3711 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3712 const auto &SeqV = OpIdx.value();
3713 if (!SeqV.isUndef())
3715 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3720 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3721 SplatValue = SignExtend64<32>(SplatValue);
3743 const auto *BV = cast<BuildVectorSDNode>(
Op);
3746 BV->getRepeatedSequence(Sequence) &&
3747 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3748 unsigned SeqLen = Sequence.size();
3750 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3751 ViaIntVT == MVT::i64) &&
3752 "Unexpected sequence type");
3757 const unsigned RequiredVL = NumElts / SeqLen;
3758 const unsigned ViaVecLen =
3760 NumElts : RequiredVL;
3763 unsigned EltIdx = 0;
3764 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3768 for (
const auto &SeqV : Sequence) {
3769 if (!SeqV.isUndef())
3771 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3777 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3778 SplatValue = SignExtend64<32>(SplatValue);
3784 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3785 "Unexpected bitcast sequence");
3786 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3789 MVT ViaContainerVT =
3796 if (ViaVecLen != RequiredVL)
3813 if (EltBitSize - SignBits < 8) {
3817 Source, DAG, Subtarget);
3838 return RISCV::PACKH;
3840 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
3855 MVT VT =
Op.getSimpleValueType();
3863 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
3868 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
3882 if (Subtarget.hasStdExtZbkb())
3887 ElemDL, XLenVT,
A,
B),
3894 Flags.setDisjoint(
true);
3900 NewOperands.
reserve(NumElts / 2);
3902 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
3912 MVT VT =
Op.getSimpleValueType();
3923 MVT VT =
Op.getSimpleValueType();
3928 !Subtarget.hasStdExtZfhmin())
4000 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
4004 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4027 unsigned NumUndefElts =
4029 unsigned NumDefElts = NumElts - NumUndefElts;
4030 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4037 for (
unsigned i = 0; i < NumElts; i++) {
4039 if (i < NumElts / 2) {
4046 bool SelectMaskVal = (i < NumElts / 2);
4049 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4050 MaskVals.
size() == NumElts);
4085 unsigned UndefCount = 0;
4092 LinearBudget -= PerSlideCost;
4095 LinearBudget -= PerSlideCost;
4098 LinearBudget -= PerSlideCost;
4101 if (LinearBudget < 0)
4106 "Illegal type which will result in reserved encoding");
4131 Vec,
Offset, Mask, VL, Policy);
4144 Vec,
Offset, Mask, VL, Policy);
4154 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4155 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4156 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4159 if ((LoC >> 31) == HiC)
4170 (isa<RegisterSDNode>(VL) &&
4171 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4173 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4188 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4189 Hi.getConstantOperandVal(1) == 31)
4208 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4220 bool HasPassthru = Passthru && !Passthru.
isUndef();
4221 if (!HasPassthru && !Passthru)
4229 if (Scalar.getValueType().bitsLE(XLenVT)) {
4236 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4240 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4241 "Unexpected scalar for splat lowering!");
4265 SDValue ExtractedVal = Scalar.getOperand(0);
4270 MVT ExtractedContainerVT = ExtractedVT;
4273 DAG, ExtractedContainerVT, Subtarget);
4275 ExtractedVal, DAG, Subtarget);
4277 if (ExtractedContainerVT.
bitsLE(VT))
4292 if (!Scalar.getValueType().bitsLE(XLenVT))
4295 VT,
DL, DAG, Subtarget);
4303 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4329 if (Src != V2.getOperand(0))
4333 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4338 V2.getConstantOperandVal(1) != Mask.size())
4342 if (Mask[0] != 0 && Mask[0] != 1)
4347 for (
unsigned i = 1; i != Mask.size(); ++i)
4348 if (Mask[i] != Mask[i - 1] + 2)
4366 int Size = Mask.size();
4368 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4374 EvenSrc = StartIndexes[0];
4375 OddSrc = StartIndexes[1];
4378 if (EvenSrc != 0 && OddSrc != 0)
4388 int HalfNumElts = NumElts / 2;
4389 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4405 int Size = Mask.size();
4417 for (
int i = 0; i !=
Size; ++i) {
4423 int StartIdx = i - (M %
Size);
4431 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4434 Rotation = CandidateRotation;
4435 else if (Rotation != CandidateRotation)
4440 int MaskSrc = M <
Size ? 0 : 1;
4445 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4450 TargetSrc = MaskSrc;
4451 else if (TargetSrc != MaskSrc)
4458 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4459 assert((LoSrc >= 0 || HiSrc >= 0) &&
4460 "Failed to find a rotated input vector!");
4475 MVT ContainerVT = VT;
4478 assert(Src.getSimpleValueType().isFixedLengthVector());
4482 MVT SrcContainerVT =
4495 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4502 unsigned Shift = EvenElts ? 0 : EltBits;
4508 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4534 auto findNonEXTRACT_SUBVECTORParent =
4535 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4540 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4541 Offset += Parent.getConstantOperandVal(1);
4542 Parent = Parent.getOperand(0);
4544 return std::make_pair(Parent,
Offset);
4547 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4548 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4557 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4558 if (NewMask[i] == -1)
4561 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4562 NewMask[i] = NewMask[i] + V1IndexOffset;
4566 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4572 if (NewMask[0] <= 0)
4576 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4577 if (NewMask[i - 1] + 1 != NewMask[i])
4581 MVT SrcVT = Src.getSimpleValueType();
4612 int NumSubElts,
Index;
4617 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4618 SDValue InPlace = OpsSwapped ? V2 : V1;
4619 SDValue ToInsert = OpsSwapped ? V1 : V2;
4629 if (NumSubElts +
Index >= (
int)NumElts)
4643 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4655 bool OpsSwapped =
false;
4656 if (!isa<BuildVectorSDNode>(V1)) {
4657 if (!isa<BuildVectorSDNode>(V2))
4662 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4670 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4671 for (
unsigned i = S; i != E; ++i)
4672 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4678 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4679 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4682 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4684 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4689 auto OpCode = IsVSlidedown ?
4694 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4697 Splat, TrueMask, VL);
4708 MVT VecContainerVT = VecVT;
4725 MVT WideContainerVT = WideVT;
4731 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4744 }
else if (Subtarget.hasStdExtZvbb()) {
4749 OffsetVec, Passthru, Mask, VL);
4752 Interleaved, EvenV, Passthru, Mask, VL);
4760 Interleaved, OffsetVec, Passthru, Mask, VL);
4768 OddV, Passthru, Mask, VL);
4774 OddV, AllOnesVec, Passthru, Mask, VL);
4782 Interleaved, OddsMul, Passthru, Mask, VL);
4789 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4835 if (ViaEltSize > NumElts)
4844 if (ViaEltSize > NumElts)
4850 if (ViaEltSize > NumElts)
4859 MVT &RotateVT,
unsigned &RotateAmt) {
4865 unsigned NumSubElts;
4867 NumElts, NumSubElts, RotateAmt))
4870 NumElts / NumSubElts);
4932 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4935 OutMasks(VRegsPerSrc, {-1, {}});
4940 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4941 int DstVecIdx = DstIdx / ElemsPerVReg;
4942 int DstSubIdx = DstIdx % ElemsPerVReg;
4943 int SrcIdx = Mask[DstIdx];
4944 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4946 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4947 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4948 if (OutMasks[DstVecIdx].first == -1)
4949 OutMasks[DstVecIdx].first = SrcVecIdx;
4950 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4956 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4957 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4971 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4972 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4973 if (SrcVecIdx == -1)
4975 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4982 unsigned InsertIdx = DstVecIdx * NumOpElts;
4995 MVT VT =
Op.getSimpleValueType();
5010 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5034 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5035 V = V.getOperand(
Offset / OpElements);
5041 auto *Ld = cast<LoadSDNode>(V);
5051 SDValue Ops[] = {Ld->getChain(),
5065 MVT SplatVT = ContainerVT;
5068 if (SVT == MVT::f16 && !Subtarget.hasStdExtZfh()) {
5077 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5078 Ld->getPointerInfo().getWithOffset(
Offset),
5079 Ld->getOriginalAlign(),
5083 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5084 Ld->getOriginalAlign(),
5085 Ld->getMemOperand()->getFlags());
5097 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5100 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5122 if (Subtarget.hasStdExtZvkb())
5133 LoV = LoSrc == 0 ? V1 : V2;
5137 HiV = HiSrc == 0 ? V1 : V2;
5143 unsigned InvRotate = NumElts - Rotation;
5153 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5173 int EvenSrc, OddSrc;
5178 int Size = Mask.size();
5180 assert(EvenSrc >= 0 &&
"Undef source?");
5181 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5185 assert(OddSrc >= 0 &&
"Undef source?");
5186 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5195 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5204 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5233 MVT IndexContainerVT =
5238 for (
int MaskIndex : Mask) {
5239 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5248 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5258 for (
int MaskIndex : Mask) {
5259 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5260 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5262 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5277 for (
int MaskIndex : Mask) {
5278 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5282 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5314RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5316 MVT VT =
Op.getSimpleValueType();
5320 MVT ContainerVT = VT;
5323 if (
Op->isVPOpcode()) {
5324 Mask =
Op.getOperand(1);
5328 VL =
Op.getOperand(2);
5334 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5336 FloatEltVT = MVT::f32;
5343 "Expected legal float type!");
5350 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5353 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5358 if (FloatVT.
bitsGT(VT)) {
5359 if (
Op->isVPOpcode())
5360 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5369 if (!
Op->isVPOpcode())
5373 MVT ContainerFloatVT =
5376 Src, Mask, RTZRM, VL);
5383 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5387 if (
Op->isVPOpcode()) {
5396 else if (IntVT.
bitsGT(VT))
5401 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5406 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5407 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5412 unsigned Adjust = ExponentBias + (EltSize - 1);
5414 if (
Op->isVPOpcode())
5424 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5425 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5435 MVT SrcVT =
Source.getSimpleValueType();
5444 SrcVT = ContainerVT;
5457 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5474 auto *
Load = cast<LoadSDNode>(
Op);
5475 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5478 Load->getMemoryVT(),
5479 *
Load->getMemOperand()))
5483 MVT VT =
Op.getSimpleValueType();
5485 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5486 "Unexpected unaligned RVV load type");
5490 "Expecting equally-sized RVV vector types to be legal");
5492 Load->getPointerInfo(),
Load->getOriginalAlign(),
5493 Load->getMemOperand()->getFlags());
5503 auto *
Store = cast<StoreSDNode>(
Op);
5504 assert(Store &&
Store->getValue().getValueType().isVector() &&
5505 "Expected vector store");
5508 Store->getMemoryVT(),
5509 *
Store->getMemOperand()))
5516 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5517 "Unexpected unaligned RVV store type");
5521 "Expecting equally-sized RVV vector types to be legal");
5522 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5524 Store->getPointerInfo(),
Store->getOriginalAlign(),
5525 Store->getMemOperand()->getFlags());
5530 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5532 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5559 unsigned ShiftAmt, AddOpc;
5576 if (Subtarget.hasStdExtZtso()) {
5600 MVT VT =
Op.getSimpleValueType();
5602 unsigned Check =
Op.getConstantOperandVal(1);
5603 unsigned TDCMask = 0;
5631 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5636 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5638 VL =
Op.getOperand(3);
5641 VL,
Op->getFlags());
5656 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5658 MVT MaskContainerVT =
5661 VL =
Op.getOperand(3);
5666 Mask, VL,
Op->getFlags());
5669 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5674 DAG.
getUNDEF(ContainerVT), Mask, VL});
5678 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5682 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5686 DAG.
getUNDEF(ContainerVT), Mask, VL});
5702 MVT VT =
Op.getSimpleValueType();
5729 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5736 MVT ContainerVT = VT;
5744 if (
Op->isVPOpcode()) {
5745 Mask =
Op.getOperand(2);
5749 VL =
Op.getOperand(3);
5757 {X, X, DAG.getCondCode(ISD::SETOEQ),
5758 DAG.getUNDEF(ContainerVT), Mask, VL});
5766 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5767 DAG.getUNDEF(ContainerVT), Mask, VL});
5777 DAG.
getUNDEF(ContainerVT), Mask, VL);
5785#define OP_CASE(NODE) \
5787 return RISCVISD::NODE##_VL;
5788#define VP_CASE(NODE) \
5789 case ISD::VP_##NODE: \
5790 return RISCVISD::NODE##_VL;
5792 switch (
Op.getOpcode()) {
5870 case ISD::VP_CTLZ_ZERO_UNDEF:
5873 case ISD::VP_CTTZ_ZERO_UNDEF:
5882 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5887 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5892 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5895 case ISD::VP_SELECT:
5904 case ISD::VP_SIGN_EXTEND:
5906 case ISD::VP_ZERO_EXTEND:
5908 case ISD::VP_FP_TO_SINT:
5910 case ISD::VP_FP_TO_UINT:
5913 case ISD::VP_FMINNUM:
5916 case ISD::VP_FMAXNUM:
5921 case ISD::VP_LLRINT:
5933 "not a RISC-V target specific op");
5939 "adding target specific op should update this function");
5959 "not a RISC-V target specific op");
5965 "adding target specific op should update this function");
5984 if (!
Op.getOperand(j).getValueType().isVector()) {
5985 LoOperands[j] =
Op.getOperand(j);
5986 HiOperands[j] =
Op.getOperand(j);
5989 std::tie(LoOperands[j], HiOperands[j]) =
5994 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5996 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6011 std::tie(LoOperands[j], HiOperands[j]) =
6015 if (!
Op.getOperand(j).getValueType().isVector()) {
6016 LoOperands[j] =
Op.getOperand(j);
6017 HiOperands[j] =
Op.getOperand(j);
6020 std::tie(LoOperands[j], HiOperands[j]) =
6025 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6027 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6037 auto [EVLLo, EVLHi] =
6038 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6042 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6044 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6062 if (!
Op.getOperand(j).getValueType().isVector()) {
6063 LoOperands[j] =
Op.getOperand(j);
6064 HiOperands[j] =
Op.getOperand(j);
6067 std::tie(LoOperands[j], HiOperands[j]) =
6072 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6075 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6084 switch (
Op.getOpcode()) {
6090 return lowerGlobalAddress(
Op, DAG);
6092 return lowerBlockAddress(
Op, DAG);
6094 return lowerConstantPool(
Op, DAG);
6096 return lowerJumpTable(
Op, DAG);
6098 return lowerGlobalTLSAddress(
Op, DAG);
6102 return lowerSELECT(
Op, DAG);
6104 return lowerBRCOND(
Op, DAG);
6106 return lowerVASTART(
Op, DAG);
6108 return lowerFRAMEADDR(
Op, DAG);
6110 return lowerRETURNADDR(
Op, DAG);
6112 return lowerShiftLeftParts(
Op, DAG);
6114 return lowerShiftRightParts(
Op, DAG,
true);
6116 return lowerShiftRightParts(
Op, DAG,
false);
6119 if (
Op.getValueType().isFixedLengthVector()) {
6120 assert(Subtarget.hasStdExtZvkb());
6121 return lowerToScalableOp(
Op, DAG);
6123 assert(Subtarget.hasVendorXTHeadBb() &&
6124 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6125 "Unexpected custom legalization");
6127 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6132 EVT VT =
Op.getValueType();
6136 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6142 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6143 Subtarget.hasStdExtZfbfmin()) {
6148 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6155 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6172 "Unexpected types");
6206 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6208 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6210 return LowerINTRINSIC_VOID(
Op, DAG);
6212 return LowerIS_FPCLASS(
Op, DAG);
6214 MVT VT =
Op.getSimpleValueType();
6216 assert(Subtarget.hasStdExtZvbb());
6217 return lowerToScalableOp(
Op, DAG);
6220 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6228 if (!
Op.getSimpleValueType().isVector())
6230 return lowerVectorTruncLike(
Op, DAG);
6233 if (
Op.getOperand(0).getValueType().isVector() &&
6234 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6235 return lowerVectorMaskExt(
Op, DAG, 1);
6238 if (
Op.getOperand(0).getValueType().isVector() &&
6239 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6240 return lowerVectorMaskExt(
Op, DAG, -1);
6243 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6245 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6247 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6249 MVT VT =
Op.getSimpleValueType();
6257 MVT ContainerVT = VT;
6263 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6270 MVT VT =
Op.getSimpleValueType();
6290 }
else if ((Val % 8) == 0) {
6306 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6307 Op.getOperand(1).getValueType() == MVT::i32) {
6319 if (
Op.getValueType() == MVT::nxv32f16 &&
6326 EVT VT =
Op.getValueType();
6329 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6331 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6337 if (!
Op.getValueType().isVector())
6339 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6343 EVT VT =
Op.getValueType();
6346 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6348 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6356 if (!
Op.getValueType().isVector())
6358 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6362 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6365 if (
Op.getValueType().isVector() &&
6366 Op.getValueType().getScalarType() == MVT::f16 &&
6369 if (
Op.getValueType() == MVT::nxv32f16)
6384 Op1.getValueType().isVector() &&
6385 Op1.getValueType().getScalarType() == MVT::f16 &&
6388 if (Op1.getValueType() == MVT::nxv32f16)
6393 Op1.getValueType().getVectorElementCount());
6396 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6406 MVT VT =
Op.getSimpleValueType();
6410 bool IsStrict =
Op->isStrictFPOpcode();
6411 SDValue Src =
Op.getOperand(0 + IsStrict);
6413 MVT SrcVT = Src.getSimpleValueType();
6418 "Unexpected vector element types");
6422 if (EltSize > (2 * SrcEltSize)) {
6434 Op.getOperand(0), Ext);
6438 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6443 auto [FExt, Chain] =
6445 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6452 if (SrcEltSize > (2 * EltSize)) {
6455 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6460 Op.getOperand(0), Src);
6475 Op.getOperand(0), Src);
6489 unsigned RVVOpc = 0;
6490 switch (
Op.getOpcode()) {
6522 "Expected same element count");
6529 Op.getOperand(0), Src, Mask, VL);
6533 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6548 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6555 MVT VT =
Op.getSimpleValueType();
6577 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6593 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6613 return lowerVECREDUCE(
Op, DAG);
6617 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6618 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6619 return lowerVECREDUCE(
Op, DAG);
6626 return lowerFPVECREDUCE(
Op, DAG);
6627 case ISD::VP_REDUCE_ADD:
6628 case ISD::VP_REDUCE_UMAX:
6629 case ISD::VP_REDUCE_SMAX:
6630 case ISD::VP_REDUCE_UMIN:
6631 case ISD::VP_REDUCE_SMIN:
6632 case ISD::VP_REDUCE_FADD:
6633 case ISD::VP_REDUCE_SEQ_FADD:
6634 case ISD::VP_REDUCE_FMIN:
6635 case ISD::VP_REDUCE_FMAX:
6636 case ISD::VP_REDUCE_FMINIMUM:
6637 case ISD::VP_REDUCE_FMAXIMUM:
6638 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6642 return lowerVPREDUCE(
Op, DAG);
6643 case ISD::VP_REDUCE_AND:
6644 case ISD::VP_REDUCE_OR:
6645 case ISD::VP_REDUCE_XOR:
6646 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6647 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6648 return lowerVPREDUCE(
Op, DAG);
6649 case ISD::VP_CTTZ_ELTS:
6650 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6651 return lowerVPCttzElements(
Op, DAG);
6655 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6658 return lowerINSERT_SUBVECTOR(
Op, DAG);
6660 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6662 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6664 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6666 return lowerSTEP_VECTOR(
Op, DAG);
6668 return lowerVECTOR_REVERSE(
Op, DAG);
6670 return lowerVECTOR_SPLICE(
Op, DAG);
6674 if ((
Op.getValueType().getScalarType() == MVT::f16 &&
6678 (
Op.getValueType().getScalarType() == MVT::bf16 &&
6680 Subtarget.hasStdExtZfbfmin()))) {
6681 if (
Op.getValueType() == MVT::nxv32f16 ||
6682 Op.getValueType() == MVT::nxv32bf16)
6694 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6695 return lowerVectorMaskSplat(
Op, DAG);
6703 MVT VT =
Op.getSimpleValueType();
6704 MVT ContainerVT = VT;
6722 Op->ops().take_front(HalfNumOps));
6724 Op->ops().drop_front(HalfNumOps));
6728 unsigned NumOpElts =
6729 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6732 SDValue SubVec = OpIdx.value();
6743 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6745 if (
Op.getValueType().isFixedLengthVector())
6746 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6749 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6751 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6752 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6756 return lowerMaskedLoad(
Op, DAG);
6759 return lowerMaskedStore(
Op, DAG);
6768 EVT VT =
Op.getValueType();
6779 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6781 MVT VT =
Op.getSimpleValueType();
6786 "Unexpected CondCode");
6794 if (isa<ConstantSDNode>(
RHS)) {
6795 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6796 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6815 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6820 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6836 return lowerToScalableOp(
Op, DAG);
6840 if (
Op.getSimpleValueType().isFixedLengthVector())
6841 return lowerToScalableOp(
Op, DAG);
6843 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6844 "Unexpected custom legalisation");
6856 if (
Op.getValueType() == MVT::nxv32f16 &&
6869 return lowerToScalableOp(
Op, DAG);
6872 return lowerToScalableOp(
Op, DAG);
6875 return lowerToScalableOp(
Op, DAG);
6879 EVT VT =
Op->getValueType(0);
6894 return lowerABS(
Op, DAG);
6899 if (Subtarget.hasStdExtZvbb())
6900 return lowerToScalableOp(
Op, DAG);
6902 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6904 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6906 if (
Op.getValueType() == MVT::nxv32f16 &&
6910 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6917 if (
Op.getValueType() == MVT::nxv32f16 &&
6921 return lowerToScalableOp(
Op, DAG);
6924 return lowerVectorStrictFSetcc(
Op, DAG);
6934 case ISD::VP_GATHER:
6935 return lowerMaskedGather(
Op, DAG);
6937 case ISD::VP_SCATTER:
6938 return lowerMaskedScatter(
Op, DAG);
6940 return lowerGET_ROUNDING(
Op, DAG);
6942 return lowerSET_ROUNDING(
Op, DAG);
6944 return lowerEH_DWARF_CFA(
Op, DAG);
6945 case ISD::VP_SELECT:
6954 case ISD::VP_UADDSAT:
6955 case ISD::VP_USUBSAT:
6956 case ISD::VP_SADDSAT:
6957 case ISD::VP_SSUBSAT:
6959 case ISD::VP_LLRINT:
6960 return lowerVPOp(
Op, DAG);
6964 return lowerLogicVPOp(
Op, DAG);
6973 case ISD::VP_FMINNUM:
6974 case ISD::VP_FMAXNUM:
6975 case ISD::VP_FCOPYSIGN:
6976 if (
Op.getValueType() == MVT::nxv32f16 &&
6984 return lowerVPOp(
Op, DAG);
6985 case ISD::VP_IS_FPCLASS:
6986 return LowerIS_FPCLASS(
Op, DAG);
6987 case ISD::VP_SIGN_EXTEND:
6988 case ISD::VP_ZERO_EXTEND:
6989 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6990 return lowerVPExtMaskOp(
Op, DAG);
6991 return lowerVPOp(
Op, DAG);
6992 case ISD::VP_TRUNCATE:
6993 return lowerVectorTruncLike(
Op, DAG);
6994 case ISD::VP_FP_EXTEND:
6995 case ISD::VP_FP_ROUND:
6996 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6997 case ISD::VP_SINT_TO_FP:
6998 case ISD::VP_UINT_TO_FP:
6999 if (
Op.getValueType().isVector() &&
7000 Op.getValueType().getScalarType() == MVT::f16 &&
7003 if (
Op.getValueType() == MVT::nxv32f16)
7015 case ISD::VP_FP_TO_SINT:
7016 case ISD::VP_FP_TO_UINT:
7018 Op1.getValueType().isVector() &&
7019 Op1.getValueType().getScalarType() == MVT::f16 &&
7022 if (Op1.getValueType() == MVT::nxv32f16)
7027 Op1.getValueType().getVectorElementCount());
7031 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7033 return lowerVPFPIntConvOp(
Op, DAG);
7035 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
7039 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7040 return lowerVPSetCCMaskOp(
Op, DAG);
7046 case ISD::VP_BITREVERSE:
7048 return lowerVPOp(
Op, DAG);
7050 case ISD::VP_CTLZ_ZERO_UNDEF:
7051 if (Subtarget.hasStdExtZvbb())
7052 return lowerVPOp(
Op, DAG);
7053 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7055 case ISD::VP_CTTZ_ZERO_UNDEF:
7056 if (Subtarget.hasStdExtZvbb())
7057 return lowerVPOp(
Op, DAG);
7058 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7060 return lowerVPOp(
Op, DAG);
7061 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7062 return lowerVPStridedLoad(
Op, DAG);
7063 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7064 return lowerVPStridedStore(
Op, DAG);
7066 case ISD::VP_FFLOOR:
7068 case ISD::VP_FNEARBYINT:
7069 case ISD::VP_FROUND:
7070 case ISD::VP_FROUNDEVEN:
7071 case ISD::VP_FROUNDTOZERO:
7072 if (
Op.getValueType() == MVT::nxv32f16 &&
7077 case ISD::VP_FMAXIMUM:
7078 case ISD::VP_FMINIMUM:
7079 if (
Op.getValueType() == MVT::nxv32f16 &&
7084 case ISD::EXPERIMENTAL_VP_SPLICE:
7085 return lowerVPSpliceExperimental(
Op, DAG);
7086 case ISD::EXPERIMENTAL_VP_REVERSE:
7087 return lowerVPReverseExperimental(
Op, DAG);
7088 case ISD::EXPERIMENTAL_VP_SPLAT:
7089 return lowerVPSplatExperimental(
Op, DAG);
7092 "llvm.clear_cache only needs custom lower on Linux targets");
7095 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7096 Op.getOperand(2), Flags,
DL);
7104 MakeLibCallOptions CallOptions;
7105 std::pair<SDValue, SDValue> CallResult =
7106 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7107 {Start,
End, Flags}, CallOptions,
DL, InChain);
7110 return CallResult.second;
7127 N->getOffset(), Flags);
7135template <
class NodeTy>
7137 bool IsLocal,
bool IsExternWeak)
const {
7147 if (IsLocal && !Subtarget.allowTaggedGlobals())
7209 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7218 return getAddr(
N, DAG);
7225 return getAddr(
N, DAG);
7232 return getAddr(
N, DAG);
7237 bool UseGOT)
const {
7301 Args.push_back(Entry);
7334 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7348 Addr = getStaticTLSAddr(
N, DAG,
false);
7351 Addr = getStaticTLSAddr(
N, DAG,
true);
7356 : getDynamicTLSAddr(
N, DAG);
7373 if (
LHS == LHS2 &&
RHS == RHS2) {
7378 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7386 return std::nullopt;
7394 MVT VT =
N->getSimpleValueType(0);
7424 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7427 if (~TrueVal == FalseVal) {
7467 if (Subtarget.hasShortForwardBranchOpt())
7470 unsigned SelOpNo = 0;
7480 unsigned ConstSelOpNo = 1;
7481 unsigned OtherSelOpNo = 2;
7482 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7487 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7488 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7492 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7493 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7499 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7501 std::swap(NewConstOps[0], NewConstOps[1]);
7513 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7515 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7518 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7519 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7528 MVT VT =
Op.getSimpleValueType();
7542 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7570 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7574 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7576 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7577 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7579 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7584 DL, VT, LHSVal, CondV);
7600 if (
Op.hasOneUse()) {
7601 unsigned UseOpc =
Op->use_begin()->getOpcode();
7610 return lowerSELECT(NewSel, DAG);
7639 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7660 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7664 if (TrueVal - 1 == FalseVal)
7666 if (TrueVal + 1 == FalseVal)
7673 RHS == TrueV && LHS == FalseV) {
7690 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7716 LHS, RHS, TargetCC,
Op.getOperand(2));
7734 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7746 int XLenInBytes = Subtarget.
getXLen() / 8;
7748 EVT VT =
Op.getValueType();
7751 unsigned Depth =
Op.getConstantOperandVal(0);
7753 int Offset = -(XLenInBytes * 2);
7769 int XLenInBytes = Subtarget.
getXLen() / 8;
7774 EVT VT =
Op.getValueType();
7776 unsigned Depth =
Op.getConstantOperandVal(0);
7778 int Off = -XLenInBytes;
7779 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7798 EVT VT =
Lo.getValueType();
7837 EVT VT =
Lo.getValueType();
7888 MVT VT =
Op.getSimpleValueType();
7914 MVT VecVT =
Op.getSimpleValueType();
7916 "Unexpected SPLAT_VECTOR_PARTS lowering");
7922 MVT ContainerVT = VecVT;
7942 int64_t ExtTrueVal)
const {
7944 MVT VecVT =
Op.getSimpleValueType();
7947 assert(Src.getValueType().isVector() &&
7948 Src.getValueType().getVectorElementType() == MVT::i1);
7969 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7971 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
7974 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
7979SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
7981 MVT ExtVT =
Op.getSimpleValueType();
7985 MVT VT =
Op.getOperand(0).getSimpleValueType();
8011 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8013 EVT MaskVT =
Op.getValueType();
8016 "Unexpected type for vector mask lowering");
8018 MVT VecVT = Src.getSimpleValueType();
8022 VL =
Op.getOperand(2);
8025 MVT ContainerVT = VecVT;
8031 MVT MaskContainerVT =
8038 std::tie(Mask, VL) =
8046 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8048 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8052 DAG.
getUNDEF(ContainerVT), Mask, VL);
8055 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8063 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8066 MVT VT =
Op.getSimpleValueType();
8068 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8072 return lowerVectorMaskTruncLike(
Op, DAG);
8080 MVT SrcVT = Src.getSimpleValueType();
8085 "Unexpected vector truncate lowering");
8087 MVT ContainerVT = SrcVT;
8091 VL =
Op.getOperand(2);
8104 std::tie(Mask, VL) =
8113 }
while (SrcEltVT != DstEltVT);
8122RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8127 MVT VT =
Op.getSimpleValueType();
8128 MVT SrcVT = Src.getSimpleValueType();
8129 MVT ContainerVT = VT;
8152 Chain, Src, Mask, VL);
8153 Chain = Src.getValue(1);
8160 Chain, Src, Mask, VL);
8171RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8174 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8181 MVT VT =
Op.getSimpleValueType();
8183 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8186 MVT SrcVT = Src.getSimpleValueType();
8188 bool IsDirectExtend =
8196 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8199 MVT ContainerVT = VT;
8203 VL =
Op.getOperand(2);
8217 std::tie(Mask, VL) =
8223 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8229 unsigned InterConvOpc =
8234 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8236 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8247static std::optional<MVT>
8253 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8255 if (MaxIdx < MinVLMAX)
8257 else if (MaxIdx < MinVLMAX * 2)
8259 else if (MaxIdx < MinVLMAX * 4)
8264 return std::nullopt;
8277 MVT VecVT =
Op.getSimpleValueType();
8291 MVT ContainerVT = VecVT;
8300 MVT OrigContainerVT = ContainerVT;
8303 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8304 const unsigned OrigIdx = IdxC->getZExtValue();
8307 DL, DAG, Subtarget)) {
8308 ContainerVT = *ShrunkVT;
8317 VLEN && ContainerVT.
bitsGT(M1VT)) {
8320 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8321 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8322 unsigned ExtractIdx =
8341 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8342 const auto *CVal = cast<ConstantSDNode>(Val);
8343 if (isInt<32>(CVal->getSExtValue())) {
8344 IsLegalInsert =
true;
8353 if (IsLegalInsert) {
8359 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8375 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8376 MVT I32ContainerVT =
8387 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8392 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8394 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8399 ValInVec, AlignedIdx);
8409 DAG.
getUNDEF(I32ContainerVT), ValLo,
8410 I32Mask, InsertI64VL);
8412 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8413 I32Mask, InsertI64VL);
8415 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8428 Idx, Mask, InsertVL, Policy);
8432 Slideup, AlignedIdx);
8447 EVT EltVT =
Op.getValueType();
8454 MVT ContainerVT = VecVT;
8470 unsigned WidenVecLen;
8473 unsigned MaxEEW = Subtarget.
getELen();
8478 "the number of elements should be power of 2");
8482 ExtractBitIdx =
Idx;
8484 WideEltVT = LargestEltVT;
8487 ExtractElementIdx = DAG.
getNode(
8498 Vec, ExtractElementIdx);
8514 MVT ContainerVT = VecVT;
8525 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8526 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8528 unsigned OrigIdx = IdxC->getZExtValue();
8531 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8532 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8533 unsigned ExtractIdx =
8543 std::optional<uint64_t> MaxIdx;
8546 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8547 MaxIdx = IdxC->getZExtValue();
8549 if (
auto SmallerVT =
8551 ContainerVT = *SmallerVT;
8598 "Unexpected opcode");
8605 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8610 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8611 if (!
II || !
II->hasScalarOperand())
8614 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
8627 if (OpVT.
bitsLT(XLenVT)) {
8634 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8644 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
8645 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8648 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8659 case Intrinsic::riscv_vslide1up:
8660 case Intrinsic::riscv_vslide1down:
8661 case Intrinsic::riscv_vslide1up_mask:
8662 case Intrinsic::riscv_vslide1down_mask: {
8665 bool IsMasked = NumOps == 7;
8671 std::tie(ScalarLo, ScalarHi) =
8679 if (isa<ConstantSDNode>(AVL)) {
8680 const auto [MinVLMAX, MaxVLMAX] =
8684 if (AVLInt <= MinVLMAX) {
8686 }
else if (AVLInt >= 2 * MaxVLMAX) {
8720 if (IntNo == Intrinsic::riscv_vslide1up ||
8721 IntNo == Intrinsic::riscv_vslide1up_mask) {
8723 ScalarHi, I32Mask, I32VL);
8725 ScalarLo, I32Mask, I32VL);
8728 ScalarLo, I32Mask, I32VL);
8730 ScalarHi, I32Mask, I32VL);
8779 const unsigned ElementWidth = 8;
8784 [[maybe_unused]]
unsigned MinVF =
8787 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8791 bool Fractional = VF < LMul1VF;
8792 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8813 MVT ContainerVT = OpVT;
8840 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8844 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8845 if (!
II || !
II->hasScalarOperand())
8848 unsigned SplatOp =
II->ScalarOperand + 1;
8861 if (OpVT.
bitsLT(XLenVT)) {
8864 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8877 EVT ValType = V.getValueType();
8878 if (ValType.isVector() && ValType.isFloatingPoint()) {
8881 ValType.getVectorElementCount());
8884 if (ValType.isFixedLengthVector()) {
8886 DAG, V.getSimpleValueType(), Subtarget);
8902 unsigned IntNo =
Op.getConstantOperandVal(0);
8909 case Intrinsic::thread_pointer: {
8913 case Intrinsic::riscv_orc_b:
8914 case Intrinsic::riscv_brev8:
8915 case Intrinsic::riscv_sha256sig0:
8916 case Intrinsic::riscv_sha256sig1:
8917 case Intrinsic::riscv_sha256sum0:
8918 case Intrinsic::riscv_sha256sum1:
8919 case Intrinsic::riscv_sm3p0:
8920 case Intrinsic::riscv_sm3p1: {
8933 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8935 case Intrinsic::riscv_sm4ks:
8936 case Intrinsic::riscv_sm4ed: {
8940 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
8943 case Intrinsic::riscv_zip:
8944 case Intrinsic::riscv_unzip: {
8947 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8949 case Intrinsic::riscv_mopr:
8953 case Intrinsic::riscv_moprr: {
8955 Op.getOperand(2),
Op.getOperand(3));
8957 case Intrinsic::riscv_clmul:
8960 case Intrinsic::riscv_clmulh:
8961 case Intrinsic::riscv_clmulr: {
8964 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
8966 case Intrinsic::experimental_get_vector_length:
8968 case Intrinsic::experimental_cttz_elts:
8970 case Intrinsic::riscv_vmv_x_s: {
8974 case Intrinsic::riscv_vfmv_f_s:
8977 case Intrinsic::riscv_vmv_v_x:
8979 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
8981 case Intrinsic::riscv_vfmv_v_f:
8983 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
8984 case Intrinsic::riscv_vmv_s_x: {
8987 if (
Scalar.getValueType().bitsLE(XLenVT)) {
8990 Op.getOperand(1), Scalar,
Op.getOperand(3));
8993 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9010 MVT VT =
Op.getSimpleValueType();
9015 if (
Op.getOperand(1).isUndef())
9031 case Intrinsic::riscv_vfmv_s_f:
9033 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9035 case Intrinsic::riscv_vaesdf_vv:
9036 case Intrinsic::riscv_vaesdf_vs:
9037 case Intrinsic::riscv_vaesdm_vv:
9038 case Intrinsic::riscv_vaesdm_vs:
9039 case Intrinsic::riscv_vaesef_vv:
9040 case Intrinsic::riscv_vaesef_vs:
9041 case Intrinsic::riscv_vaesem_vv:
9042 case Intrinsic::riscv_vaesem_vs:
9043 case Intrinsic::riscv_vaeskf1:
9044 case Intrinsic::riscv_vaeskf2:
9045 case Intrinsic::riscv_vaesz_vs:
9046 case Intrinsic::riscv_vsm4k:
9047 case Intrinsic::riscv_vsm4r_vv:
9048 case Intrinsic::riscv_vsm4r_vs: {
9049 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9050 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9051 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9056 case Intrinsic::riscv_vsm3c:
9057 case Intrinsic::riscv_vsm3me: {
9058 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9059 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9064 case Intrinsic::riscv_vsha2ch:
9065 case Intrinsic::riscv_vsha2cl:
9066 case Intrinsic::riscv_vsha2ms: {
9067 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9068 !Subtarget.hasStdExtZvknhb())
9070 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9071 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9072 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9076 case Intrinsic::riscv_sf_vc_v_x:
9077 case Intrinsic::riscv_sf_vc_v_i:
9078 case Intrinsic::riscv_sf_vc_v_xv:
9079 case Intrinsic::riscv_sf_vc_v_iv:
9080 case Intrinsic::riscv_sf_vc_v_vv:
9081 case Intrinsic::riscv_sf_vc_v_fv:
9082 case Intrinsic::riscv_sf_vc_v_xvv:
9083 case Intrinsic::riscv_sf_vc_v_ivv:
9084 case Intrinsic::riscv_sf_vc_v_vvv:
9085 case Intrinsic::riscv_sf_vc_v_fvv:
9086 case Intrinsic::riscv_sf_vc_v_xvw:
9087 case Intrinsic::riscv_sf_vc_v_ivw:
9088 case Intrinsic::riscv_sf_vc_v_vvw:
9089 case Intrinsic::riscv_sf_vc_v_fvw: {
9090 MVT VT =
Op.getSimpleValueType();
9127 MVT VT =
Op.getSimpleValueType();
9131 if (VT.isFloatingPoint()) {
9136 if (VT.isFixedLengthVector())
9146 if (VT.isFixedLengthVector())
9148 if (VT.isFloatingPoint())
9167 unsigned IntNo =
Op.getConstantOperandVal(1);
9171 case Intrinsic::riscv_seg2_load:
9172 case Intrinsic::riscv_seg3_load:
9173 case Intrinsic::riscv_seg4_load:
9174 case Intrinsic::riscv_seg5_load:
9175 case Intrinsic::riscv_seg6_load:
9176 case Intrinsic::riscv_seg7_load:
9177 case Intrinsic::riscv_seg8_load: {
9180 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9181 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9182 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9183 Intrinsic::riscv_vlseg8};
9184 unsigned NF =
Op->getNumValues() - 1;
9185 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9187 MVT VT =
Op->getSimpleValueType(0);
9192 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9194 ContainerVTs.push_back(MVT::Other);
9202 Load->getMemoryVT(),
Load->getMemOperand());
9204 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9210 case Intrinsic::riscv_sf_vc_v_x_se:
9212 case Intrinsic::riscv_sf_vc_v_i_se:
9214 case Intrinsic::riscv_sf_vc_v_xv_se:
9216 case Intrinsic::riscv_sf_vc_v_iv_se:
9218 case Intrinsic::riscv_sf_vc_v_vv_se:
9220 case Intrinsic::riscv_sf_vc_v_fv_se:
9222 case Intrinsic::riscv_sf_vc_v_xvv_se:
9224 case Intrinsic::riscv_sf_vc_v_ivv_se:
9226 case Intrinsic::riscv_sf_vc_v_vvv_se:
9228 case Intrinsic::riscv_sf_vc_v_fvv_se:
9230 case Intrinsic::riscv_sf_vc_v_xvw_se:
9232 case Intrinsic::riscv_sf_vc_v_ivw_se:
9234 case Intrinsic::riscv_sf_vc_v_vvw_se:
9236 case Intrinsic::riscv_sf_vc_v_fvw_se:
9245 unsigned IntNo =
Op.getConstantOperandVal(1);
9249 case Intrinsic::riscv_seg2_store:
9250 case Intrinsic::riscv_seg3_store:
9251 case Intrinsic::riscv_seg4_store:
9252 case Intrinsic::riscv_seg5_store:
9253 case Intrinsic::riscv_seg6_store:
9254 case Intrinsic::riscv_seg7_store:
9255 case Intrinsic::riscv_seg8_store: {
9258 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9259 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9260 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9261 Intrinsic::riscv_vsseg8};
9264 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9266 MVT VT =
Op->getOperand(2).getSimpleValueType();
9273 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9275 for (
unsigned i = 0; i < NF; i++)
9277 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9282 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9284 case Intrinsic::riscv_sf_vc_xv_se:
9286 case Intrinsic::riscv_sf_vc_iv_se:
9288 case Intrinsic::riscv_sf_vc_vv_se:
9290 case Intrinsic::riscv_sf_vc_fv_se:
9292 case Intrinsic::riscv_sf_vc_xvv_se:
9294 case Intrinsic::riscv_sf_vc_ivv_se:
9296 case Intrinsic::riscv_sf_vc_vvv_se:
9298 case Intrinsic::riscv_sf_vc_fvv_se:
9300 case Intrinsic::riscv_sf_vc_xvw_se:
9302 case Intrinsic::riscv_sf_vc_ivw_se:
9304 case Intrinsic::riscv_sf_vc_vvw_se:
9306 case Intrinsic::riscv_sf_vc_fvw_se:
9314 switch (ISDOpcode) {
9317 case ISD::VP_REDUCE_ADD:
9320 case ISD::VP_REDUCE_UMAX:
9323 case ISD::VP_REDUCE_SMAX:
9326 case ISD::VP_REDUCE_UMIN:
9329 case ISD::VP_REDUCE_SMIN:
9332 case ISD::VP_REDUCE_AND:
9335 case ISD::VP_REDUCE_OR:
9338 case ISD::VP_REDUCE_XOR:
9341 case ISD::VP_REDUCE_FADD:
9343 case ISD::VP_REDUCE_SEQ_FADD:
9345 case ISD::VP_REDUCE_FMAX:
9346 case ISD::VP_REDUCE_FMAXIMUM:
9348 case ISD::VP_REDUCE_FMIN:
9349 case ISD::VP_REDUCE_FMINIMUM:
9359 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9364 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9365 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9366 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9367 "Unexpected reduction lowering");
9371 MVT ContainerVT = VecVT;
9380 VL =
Op.getOperand(3);
9382 std::tie(Mask, VL) =
9390 switch (
Op.getOpcode()) {
9394 case ISD::VP_REDUCE_AND: {
9404 case ISD::VP_REDUCE_OR:
9411 case ISD::VP_REDUCE_XOR: {
9434 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9438 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9439 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9440 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9441 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9457 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9461 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9464 if (M1VT != InnerVT)
9470 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9489 VecEVT =
Lo.getValueType();
9502 MVT ContainerVT = VecVT;
9522 Mask, VL,
DL, DAG, Subtarget);
9528static std::tuple<unsigned, SDValue, SDValue>
9532 auto Flags =
Op->getFlags();
9533 unsigned Opcode =
Op.getOpcode();
9557 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9565 MVT VecEltVT =
Op.getSimpleValueType();
9569 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9573 MVT ContainerVT = VecVT;
9579 MVT ResVT =
Op.getSimpleValueType();
9582 VL,
DL, DAG, Subtarget);
9587 if (
Op->getFlags().hasNoNaNs())
9593 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9594 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9600 DL, ResVT, NoNaNs, Res,
9608 unsigned Opc =
Op.getOpcode();
9631 Vec, Mask, VL,
DL, DAG, Subtarget);
9632 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
9633 Op->getFlags().hasNoNaNs())
9650 DL, ResVT, NoNaNs, Res,
9664 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9673 (OrigIdx != 0 || !Vec.
isUndef())) {
9676 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9679 "Unexpected mask vector lowering");
9713 MVT ContainerVT = VecVT;
9749 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9750 SlideupAmt, Mask, VL, Policy);
9758 MVT ContainerVecVT = VecVT;
9764 MVT ContainerSubVecVT = SubVecVT;
9780 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
9781 SubRegIdx = Decompose.first;
9783 (OrigIdx % Vscale));
9787 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
9788 SubRegIdx = Decompose.first;
9795 bool ExactlyVecRegSized =
9797 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
9812 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
9816 if (SubRegIdx == RISCV::NoSubRegister) {
9835 MVT InterSubVT = ContainerVecVT;
9874 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
9875 SlideupAmt, Mask, VL, Policy);
9880 if (ContainerVecVT.
bitsGT(InterSubVT))
9889 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
9895 MVT SubVecVT =
Op.getSimpleValueType();
9900 unsigned OrigIdx =
Op.getConstantOperandVal(1);
9911 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9914 "Unexpected mask vector lowering");
9953 MVT ContainerVT = VecVT;
9963 ContainerVT = *ShrunkVT;
9976 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
9988 MVT ContainerSubVecVT = SubVecVT;
10002 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10003 SubRegIdx = Decompose.first;
10005 (OrigIdx % Vscale));
10009 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10010 SubRegIdx = Decompose.first;
10033 MVT InterSubVT = VecVT;
10037 assert(SubRegIdx != RISCV::NoSubRegister);
10050 Vec, SlidedownAmt, Mask, VL);
10059 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10066 MVT VT =
N.getSimpleValueType();
10070 assert(
Op.getSimpleValueType() == VT &&
10071 "Operands and result must be same type");
10075 unsigned NumVals =
N->getNumValues();
10078 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10081 for (
unsigned I = 0;
I < NumVals;
I++) {
10087 if (TruncVals.
size() > 1)
10089 return TruncVals.
front();
10095 MVT VecVT =
Op.getSimpleValueType();
10098 "vector_interleave on non-scalable vector!");
10109 EVT SplitVT = Op0Lo.getValueType();
10112 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10114 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10128 Op.getOperand(0),
Op.getOperand(1));
10155 Concat, EvenIdx, Passthru, Mask, VL);
10157 Concat, OddIdx, Passthru, Mask, VL);
10171 MVT VecVT =
Op.getSimpleValueType();
10174 "vector_interleave on non-scalable vector!");
10187 EVT SplitVT = Op0Lo.getValueType();
10190 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10192 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10214 Op.getOperand(0),
Op.getOperand(1));
10262 MVT VT =
Op.getSimpleValueType();
10267 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10268 if (StepValImm != 1) {
10277 VL, VT,
DL, DAG, Subtarget);
10292 MVT VecVT =
Op.getSimpleValueType();
10302 unsigned MaxVLMAX =
10312 if (MaxVLMAX > 256 && EltSize == 8) {
10359 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10369 MVT VecVT =
Op.getSimpleValueType();
10373 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10374 SDValue DownOffset, UpOffset;
10375 if (ImmValue >= 0) {
10391 DownOffset, TrueMask, UpOffset);
10392 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10398RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10401 auto *
Load = cast<LoadSDNode>(
Op);
10404 Load->getMemoryVT(),
10405 *
Load->getMemOperand()) &&
10406 "Expecting a correctly-aligned load");
10408 MVT VT =
Op.getSimpleValueType();
10414 const auto [MinVLMAX, MaxVLMAX] =
10417 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10431 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10440 Load->getMemoryVT(),
Load->getMemOperand());
10447RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10450 auto *
Store = cast<StoreSDNode>(
Op);
10453 Store->getMemoryVT(),
10454 *
Store->getMemOperand()) &&
10455 "Expecting a correctly-aligned store");
10476 const auto [MinVLMAX, MaxVLMAX] =
10479 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10490 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10493 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10494 Store->getMemoryVT(),
Store->getMemOperand());
10500 MVT VT =
Op.getSimpleValueType();
10502 const auto *MemSD = cast<MemSDNode>(
Op);
10503 EVT MemVT = MemSD->getMemoryVT();
10505 SDValue Chain = MemSD->getChain();
10509 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10510 Mask = VPLoad->getMask();
10512 VL = VPLoad->getVectorLength();
10514 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10515 Mask = MLoad->getMask();
10516 PassThru = MLoad->getPassThru();
10523 MVT ContainerVT = VT;
10537 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10554 Chain =
Result.getValue(1);
10566 const auto *MemSD = cast<MemSDNode>(
Op);
10567 EVT MemVT = MemSD->getMemoryVT();
10569 SDValue Chain = MemSD->getChain();
10573 bool IsCompressingStore =
false;
10574 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10575 Val = VPStore->getValue();
10576 Mask = VPStore->getMask();
10577 VL = VPStore->getVectorLength();
10579 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10580 Val = MStore->getValue();
10581 Mask = MStore->getMask();
10582 IsCompressingStore = MStore->isCompressingStore();
10591 MVT ContainerVT = VT;
10596 if (!IsUnmasked || IsCompressingStore) {
10605 if (IsCompressingStore) {
10608 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10615 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10624 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10628RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10630 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10633 MVT VT =
Op.getSimpleValueType();
10647 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10654 unsigned Opc =
Op.getOpcode();
10661 MVT VT =
Op.getSimpleValueType();
10694 MVT ContainerInVT = InVT;
10713 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10717 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10725 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10730 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10743 MVT VT =
Op.getSimpleValueType();
10747 "Unexpected type for ISD::ABS");
10749 MVT ContainerVT = VT;
10756 if (
Op->getOpcode() == ISD::VP_ABS) {
10757 Mask =
Op->getOperand(1);
10761 VL =
Op->getOperand(2);
10769 DAG.
getUNDEF(ContainerVT), Mask, VL);
10771 DAG.
getUNDEF(ContainerVT), Mask, VL);
10778SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
10781 MVT VT =
Op.getSimpleValueType();
10785 "Can only handle COPYSIGN with matching types.");
10794 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
10799SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
10801 MVT VT =
Op.getSimpleValueType();
10804 MVT I1ContainerVT =
10818 Op2, DAG.
getUNDEF(ContainerVT), VL);
10829 MVT VT =
Op.getSimpleValueType();
10834 for (
const SDValue &V :
Op->op_values()) {
10835 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10838 if (!
V.getValueType().isVector()) {
10844 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
10845 "Only fixed length vectors are supported!");
10859 if (
Op->isStrictFPOpcode()) {
10868 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
10882 MVT VT =
Op.getSimpleValueType();
10885 MVT ContainerVT = VT;
10891 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10894 if (HasPassthruOp) {
10897 if (*MaskIdx == OpIdx.index())
10901 if (
Op.getOpcode() == ISD::VP_MERGE) {
10905 assert(
Op.getOpcode() == ISD::VP_SELECT);
10912 if (!
V.getValueType().isFixedLengthVector()) {
10917 MVT OpVT =
V.getSimpleValueType();
10919 assert(useRVVForFixedLengthVectorVT(OpVT) &&
10920 "Only fixed length vectors are supported!");
10925 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
10935 MVT VT =
Op.getSimpleValueType();
10941 MVT ContainerVT = VT;
10951 DAG.
getUNDEF(ContainerVT), Zero, VL);
10954 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
10956 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
10959 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
10968 MVT VT =
Op.getSimpleValueType();
10972 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
10976 MVT ContainerVT = VT;
10986 switch (Condition) {
11054 MVT DstVT =
Op.getSimpleValueType();
11055 MVT SrcVT = Src.getSimpleValueType();
11068 if (DstEltSize >= SrcEltSize) {
11077 if (SrcEltSize == 1) {
11088 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11089 }
else if (DstEltSize > (2 * SrcEltSize)) {
11093 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11099 "Wrong input/output vector types");
11102 if (DstEltSize > (2 * SrcEltSize)) {
11118 MVT InterimFVT = DstVT;
11119 if (SrcEltSize > (2 * DstEltSize)) {
11120 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11127 if (InterimFVT != DstVT) {
11133 "Wrong input/output vector types");
11137 if (DstEltSize == 1) {
11140 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11150 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11160 while (InterimIVT != DstVT) {
11172 MVT VT =
Op.getSimpleValueType();
11179RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11191 MVT VT =
Op.getSimpleValueType();
11192 MVT ContainerVT = VT;
11202 if (IsMaskVector) {
11213 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11222 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11225 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11226 SDValue DownOffset, UpOffset;
11227 if (ImmValue >= 0) {
11241 Op1, DownOffset, Mask, UpOffset);
11245 if (IsMaskVector) {
11249 {Result, DAG.getConstant(0, DL, ContainerVT),
11250 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11265 MVT VT =
Op.getSimpleValueType();
11267 MVT ContainerVT = VT;
11283RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11286 MVT VT =
Op.getSimpleValueType();
11293 MVT ContainerVT = VT;
11301 MVT GatherVT = ContainerVT;
11305 if (IsMaskVector) {
11316 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11322 unsigned MaxVLMAX =
11331 if (MaxVLMAX > 256 && EltSize == 8) {
11359 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11361 if (IsMaskVector) {
11384 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11386 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11388 DAG.
getUNDEF(GatherVT), Mask, EVL);
11390 if (IsMaskVector) {
11405 MVT VT =
Op.getSimpleValueType();
11407 return lowerVPOp(
Op, DAG);
11414 MVT ContainerVT = VT;
11433 MVT VT =
Op.getSimpleValueType();
11434 MVT ContainerVT = VT;
11440 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11446 : Intrinsic::riscv_vlse_mask,
11449 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11450 VPNode->getStride()};
11458 Ops.
push_back(VPNode->getVectorLength());
11466 VPNode->getMemoryVT(), VPNode->getMemOperand());
11480 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11481 SDValue StoreVal = VPNode->getValue();
11483 MVT ContainerVT = VT;
11494 : Intrinsic::riscv_vsse_mask,
11497 VPNode->getBasePtr(), VPNode->getStride()};
11505 Ops.
push_back(VPNode->getVectorLength());
11508 Ops, VPNode->getMemoryVT(),
11509 VPNode->getMemOperand());
11521 MVT VT =
Op.getSimpleValueType();
11523 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11524 EVT MemVT = MemSD->getMemoryVT();
11526 SDValue Chain = MemSD->getChain();
11532 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11533 Index = VPGN->getIndex();
11534 Mask = VPGN->getMask();
11536 VL = VPGN->getVectorLength();
11541 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11542 Index = MGN->getIndex();
11543 Mask = MGN->getMask();
11544 PassThru = MGN->getPassThru();
11548 MVT IndexVT =
Index.getSimpleValueType();
11552 "Unexpected VTs!");
11553 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11556 "Unexpected extending MGATHER/VP_GATHER");
11562 MVT ContainerVT = VT;
11586 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11603 Chain =
Result.getValue(1);
11620 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11621 EVT MemVT = MemSD->getMemoryVT();
11623 SDValue Chain = MemSD->getChain();
11626 [[maybe_unused]]
bool IsTruncatingStore =
false;
11629 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11630 Index = VPSN->getIndex();
11631 Mask = VPSN->getMask();
11632 Val = VPSN->getValue();
11633 VL = VPSN->getVectorLength();
11635 IsTruncatingStore =
false;
11638 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11639 Index = MSN->getIndex();
11640 Mask = MSN->getMask();
11641 Val = MSN->getValue();
11642 IsTruncatingStore = MSN->isTruncatingStore();
11646 MVT IndexVT =
Index.getSimpleValueType();
11650 "Unexpected VTs!");
11651 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11654 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11660 MVT ContainerVT = VT;
11684 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11694 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11703 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11711 static const int Table =
11735 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11741 static const unsigned Table =
11764 bool isRISCV64 = Subtarget.
is64Bit();
11828 switch (
N->getOpcode()) {
11830 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
11836 "Unexpected custom legalisation");
11837 bool IsStrict =
N->isStrictFPOpcode();
11840 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
11858 Opc,
DL, VTs, Chain, Op0,
11892 std::tie(Result, Chain) =
11893 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
11921 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
11932 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
11933 "has custom type legalization on riscv32");
11935 SDValue LoCounter, HiCounter;
11939 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
11941 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
11944 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
11946 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
11950 N->getOperand(0), LoCounter, HiCounter);
11974 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
11975 unsigned XLen = Subtarget.
getXLen();
11978 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
11986 if (LHSIsU == RHSIsU)
12003 if (RHSIsU && LHSIsS && !RHSIsS)
12005 else if (LHSIsU && RHSIsS && !LHSIsS)
12015 "Unexpected custom legalisation");
12022 "Unexpected custom legalisation");
12025 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12051 "Unexpected custom legalisation");
12052 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12053 Subtarget.hasVendorXTHeadBb()) &&
12054 "Unexpected custom legalization");
12055 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12056 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12065 "Unexpected custom legalisation");
12079 MVT VT =
N->getSimpleValueType(0);
12080 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12081 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12082 "Unexpected custom legalisation");
12094 if (VT != MVT::i32)
12103 "Unexpected custom legalisation");
12107 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12124 EVT OType =
N->getValueType(1);
12137 "Unexpected custom legalisation");
12154 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12158 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12176 !Subtarget.hasStdExtZbb() &&
"Unexpected custom legalisation");
12185 "Unexpected custom legalisation");
12191 "Unexpected custom legalisation");
12193 if (Subtarget.hasStdExtZbb()) {
12227 EVT VT =
N->getValueType(0);
12232 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12236 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12237 Subtarget.hasStdExtZfbfmin()) {
12240 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12245 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12247 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12268 MVT VT =
N->getSimpleValueType(0);
12270 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12271 "Unexpected custom legalisation");
12274 "Unexpected extension");
12300 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12302 "Unexpected EXTRACT_VECTOR_ELT legalization");
12305 MVT ContainerVT = VecVT;
12333 DAG.
getUNDEF(ContainerVT), Mask, VL);
12341 unsigned IntNo =
N->getConstantOperandVal(0);
12345 "Don't know how to custom type legalize this intrinsic!");
12346 case Intrinsic::experimental_get_vector_length: {
12351 case Intrinsic::experimental_cttz_elts: {
12357 case Intrinsic::riscv_orc_b:
12358 case Intrinsic::riscv_brev8:
12359 case Intrinsic::riscv_sha256sig0:
12360 case Intrinsic::riscv_sha256sig1:
12361 case Intrinsic::riscv_sha256sum0:
12362 case Intrinsic::riscv_sha256sum1:
12363 case Intrinsic::riscv_sm3p0:
12364 case Intrinsic::riscv_sm3p1: {
12365 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12385 case Intrinsic::riscv_sm4ks:
12386 case Intrinsic::riscv_sm4ed: {
12394 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12398 case Intrinsic::riscv_mopr: {
12399 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12409 case Intrinsic::riscv_moprr: {
12410 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12422 case Intrinsic::riscv_clmul: {
12423 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12434 case Intrinsic::riscv_clmulh:
12435 case Intrinsic::riscv_clmulr: {
12436 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12464 case Intrinsic::riscv_vmv_x_s: {
12465 EVT VT =
N->getValueType(0);
12467 if (VT.
bitsLT(XLenVT)) {
12476 "Unexpected custom legalization");
12514 case ISD::VP_REDUCE_ADD:
12515 case ISD::VP_REDUCE_AND:
12516 case ISD::VP_REDUCE_OR:
12517 case ISD::VP_REDUCE_XOR:
12518 case ISD::VP_REDUCE_SMAX:
12519 case ISD::VP_REDUCE_UMAX:
12520 case ISD::VP_REDUCE_SMIN:
12521 case ISD::VP_REDUCE_UMIN:
12585 const EVT VT =
N->getValueType(0);
12586 const unsigned Opc =
N->getOpcode();
12593 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12598 "Inconsistent mappings");
12609 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12612 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12627 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12629 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12630 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12634 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12641 if (
LHS.getOpcode() != ReduceOpc)
12656 auto Flags = ReduceVec->
getFlags();
12657 Flags.intersectWith(
N->getFlags());
12658 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12668 auto BinOpToRVVReduce = [](
unsigned Opc) {
12697 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12700 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12703 unsigned Opc =
N->getOpcode();
12704 unsigned ReduceIdx;
12705 if (IsReduction(
N->getOperand(0), Opc))
12707 else if (IsReduction(
N->getOperand(1), Opc))
12713 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12716 SDValue Extract =
N->getOperand(ReduceIdx);
12748 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
12775 if (!Subtarget.hasStdExtZba())
12779 EVT VT =
N->getValueType(0);
12791 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12792 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
12795 int64_t C0 = N0C->getSExtValue();
12796 int64_t C1 = N1C->getSExtValue();
12797 if (C0 <= 0 || C1 <= 0)
12801 int64_t Bits = std::min(C0, C1);
12802 int64_t Diff = std::abs(C0 - C1);
12803 if (Diff != 1 && Diff != 2 && Diff != 3)
12830 EVT VT =
N->getValueType(0);
12838 if ((!Subtarget.hasStdExtZicond() &&
12839 !Subtarget.hasVendorXVentanaCondOps()) ||
12861 bool SwapSelectOps;
12867 SwapSelectOps =
false;
12868 NonConstantVal = FalseVal;
12870 SwapSelectOps =
true;
12871 NonConstantVal = TrueVal;
12877 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
12925 EVT VT =
N->getValueType(0);
12933 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12934 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12940 if (!N0C->hasOneUse())
12942 int64_t C0 = N0C->getSExtValue();
12943 int64_t C1 = N1C->getSExtValue();
12945 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
12948 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
12949 !isInt<12>(C0 * (C1 / C0))) {
12952 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
12953 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
12956 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
12957 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
12984 EVT VT =
N->getValueType(0);
13015 unsigned OuterExtend =
13019 OuterExtend,
SDLoc(
N), VT,
13027 EVT VT =
N->getValueType(0);
13075 EVT VT =
N->getValueType(0);
13079 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13085 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13095 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13116 if (!Subtarget.hasStdExtZbb())
13119 EVT VT =
N->getValueType(0);
13121 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
13130 auto *ShAmtC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
13131 if (!ShAmtC || ShAmtC->getZExtValue() != 8)
13146 EVT VT =
N->getValueType(0);
13177 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13201 EVT VT =
N->getValueType(0);
13225 EVT VT =
N->getValueType(0);
13252 if (CondLHS != True)
13259 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
13271 if (!FalseRHSC || !FalseRHSC->
isZero())
13291 EVT VT =
N->getValueType(0);
13298 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
13325 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
13377 EVT VT =
N->getValueType(0);
13423 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
13448 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13453 const APInt &Imm = ConstN00->getAPIntValue();
13454 if ((Imm + 1).isSignedIntN(12))
13475 EVT VT =
N->getValueType(0);
13487 const bool HasShlAdd =
13488 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
13502 for (
uint64_t Divisor : {3, 5, 9}) {
13503 if (MulAmt % Divisor != 0)
13505 uint64_t MulAmt2 = MulAmt / Divisor;
13512 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
13513 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
13530 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13546 if (ScaleShift >= 1 && ScaleShift < 4) {
13547 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13561 for (
uint64_t Divisor : {3, 5, 9}) {
13566 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13577 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13579 if (ScaleShift >= 1 && ScaleShift < 4) {
13580 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13606 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
13608 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
13625 EVT VT =
N->getValueType(0);
13632 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
13633 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
13646 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
13647 V3 != (HalfSize - 1))
13663 EVT VT =
N->getValueType(0);
13671 unsigned AddSubOpc;
13677 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13678 AddSubOpc = V->getOpcode();
13680 SDValue Opnd = V->getOperand(1);
13681 MulOper = V->getOperand(0);
13690 if (IsAddSubWith1(N0)) {
13692 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13695 if (IsAddSubWith1(N1)) {
13697 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13712 if (isIndexTypeSigned(IndexType))
13715 if (!
N->hasOneUse())
13718 EVT VT =
N.getValueType();
13757 EVT SrcVT = Src.getValueType();
13761 NewElen = std::max(NewElen, 8U);
13784 EVT VT =
N->getValueType(0);
13787 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
13791 auto *N1C = dyn_cast<ConstantSDNode>(N1);
13803 if (!isIntEqualitySetCC(
Cond))
13812 const APInt &C1 = N1C->getAPIntValue();
13830 EVT VT =
N->getValueType(0);
13834 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
13836 Src.getOperand(0));
13844struct CombineResult;
13846enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
13873struct NodeExtensionHelper {
13882 bool SupportsFPExt;
13885 bool EnforceOneUse;
13900 return OrigOperand;
13911 unsigned getExtOpc(ExtKind SupportsExt)
const {
13912 switch (SupportsExt) {
13913 case ExtKind::SExt:
13915 case ExtKind::ZExt:
13917 case ExtKind::FPExt:
13928 std::optional<ExtKind> SupportsExt)
const {
13929 if (!SupportsExt.has_value())
13930 return OrigOperand;
13932 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
13936 if (
Source.getValueType() == NarrowVT)
13939 unsigned ExtOpc = getExtOpc(*SupportsExt);
13943 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
13950 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
13962 DAG.
getUNDEF(NarrowVT), Source, VL);
13975 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
13981 MVT EltVT = SupportsExt == ExtKind::FPExt
13983 :
MVT::getIntegerVT(NarrowSize);
13985 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
13986 "Trying to extend something we can't represent");
13993 static unsigned getSExtOpcode(
unsigned Opcode) {
14016 static unsigned getZExtOpcode(
unsigned Opcode) {
14042 static unsigned getFPExtOpcode(
unsigned Opcode) {
14067 static unsigned getSUOpcode(
unsigned Opcode) {
14069 "SU is only supported for MUL");
14075 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14095 using CombineToTry = std::function<std::optional<CombineResult>(
14096 SDNode * ,
const NodeExtensionHelper & ,
14101 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14105 unsigned Opc = OrigOperand.
getOpcode();
14109 "Unexpected Opcode");
14122 unsigned ScalarBits =
Op.getValueSizeInBits();
14124 if (ScalarBits < EltBits) {
14127 !Subtarget.
is64Bit() &&
"Unexpected splat");
14129 SupportsSExt =
true;
14133 SupportsZExt =
true;
14135 EnforceOneUse =
false;
14139 unsigned NarrowSize = EltBits / 2;
14142 if (NarrowSize < 8)
14146 SupportsSExt =
true;
14150 SupportsZExt =
true;
14152 EnforceOneUse =
false;
14159 SupportsZExt =
false;
14160 SupportsSExt =
false;
14161 SupportsFPExt =
false;
14162 EnforceOneUse =
true;
14163 unsigned Opc = OrigOperand.
getOpcode();
14185 SupportsZExt =
true;
14188 SupportsSExt =
true;
14191 SupportsFPExt =
true;
14195 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14208 unsigned ScalarBits =
Op.getOperand(0).getValueSizeInBits();
14209 if (NarrowSize != ScalarBits)
14212 SupportsFPExt =
true;
14221 static bool isSupportedRoot(
const SDNode *Root,
14250 Subtarget.hasStdExtZvbb();
14252 return Subtarget.hasStdExtZvbb();
14266 assert(isSupportedRoot(Root, Subtarget) &&
14267 "Trying to build an helper with an "
14268 "unsupported root");
14269 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14285 if (OperandIdx == 1) {
14294 EnforceOneUse =
false;
14299 fillUpExtensionSupport(Root, DAG, Subtarget);
14305 static std::pair<SDValue, SDValue>
14308 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14327 switch (
N->getOpcode()) {
14369struct CombineResult {
14371 unsigned TargetOpcode;
14373 std::optional<ExtKind> LHSExt;
14374 std::optional<ExtKind> RHSExt;
14378 NodeExtensionHelper
LHS;
14380 NodeExtensionHelper
RHS;
14382 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14383 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14384 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14385 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14394 std::tie(Mask, VL) =
14395 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14409 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14410 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14411 Passthru, Mask, VL);
14425static std::optional<CombineResult>
14426canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14427 const NodeExtensionHelper &RHS,
14430 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14431 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14432 Root, LHS, {ExtKind::ZExt}, RHS,
14434 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14435 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14436 Root, LHS, {ExtKind::SExt}, RHS,
14438 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14439 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14440 Root, LHS, {ExtKind::FPExt}, RHS,
14442 return std::nullopt;
14451static std::optional<CombineResult>
14452canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14455 return canFoldToVWWithSameExtensionImpl(
14456 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14464static std::optional<CombineResult>
14465canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14468 if (
RHS.SupportsFPExt)
14469 return CombineResult(
14470 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14471 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14478 return CombineResult(
14479 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14480 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14482 return CombineResult(
14483 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14484 LHS, std::nullopt, RHS, {ExtKind::SExt});
14485 return std::nullopt;
14492static std::optional<CombineResult>
14493canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14496 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14504static std::optional<CombineResult>
14505canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14508 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14516static std::optional<CombineResult>
14517canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14520 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14528static std::optional<CombineResult>
14529canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14533 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14534 return std::nullopt;
14535 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14536 Root, LHS, {ExtKind::SExt}, RHS,
14541NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14552 Strategies.
push_back(canFoldToVWWithSameExtension);
14561 Strategies.
push_back(canFoldToVWWithSameExtension);
14566 Strategies.
push_back(canFoldToVWWithSameExtension);
14573 Strategies.
push_back(canFoldToVWWithZEXT);
14578 Strategies.
push_back(canFoldToVWWithSEXT);
14583 Strategies.
push_back(canFoldToVWWithZEXT);
14588 Strategies.
push_back(canFoldToVWWithFPEXT);
14617 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14623 Inserted.insert(
N);
14626 while (!Worklist.
empty()) {
14629 NodeExtensionHelper
LHS(Root, 0, DAG, Subtarget);
14630 NodeExtensionHelper
RHS(Root, 1, DAG, Subtarget);
14631 auto AppendUsersIfNeeded = [&Worklist, &Subtarget,
14632 &Inserted](
const NodeExtensionHelper &
Op) {
14633 if (
Op.needToPromoteOtherUsers()) {
14635 UE =
Op.OrigOperand->use_end();
14638 if (!NodeExtensionHelper::isSupportedRoot(TheUse, Subtarget))
14641 if (UI.getOperandNo() >= 2)
14643 if (Inserted.insert(TheUse).second)
14656 NodeExtensionHelper::getSupportedFoldings(Root);
14658 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14659 bool Matched =
false;
14660 for (
int Attempt = 0;
14661 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
14664 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14665 FoldingStrategies) {
14666 std::optional<CombineResult> Res =
14667 FoldingStrategy(Root,
LHS,
RHS, DAG, Subtarget);
14674 if (Res->LHSExt.has_value())
14675 if (!AppendUsersIfNeeded(
LHS))
14677 if (Res->RHSExt.has_value())
14678 if (!AppendUsersIfNeeded(
RHS))
14690 SDValue InputRootReplacement;
14697 for (CombineResult Res : CombinesToApply) {
14698 SDValue NewValue = Res.materialize(DAG, Subtarget);
14699 if (!InputRootReplacement) {
14701 "First element is expected to be the current node");
14702 InputRootReplacement = NewValue;
14707 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14711 return InputRootReplacement;
14718 unsigned Opc =
N->getOpcode();
14723 SDValue MergeOp =
N->getOperand(1);
14724 unsigned MergeOpc = MergeOp.
getOpcode();
14735 SDValue Passthru =
N->getOperand(2);
14749 Z = Z.getOperand(1);
14755 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14762 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14791 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14797 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14799 if (MemVT == MVT::i32)
14805 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
14840 if (!Subtarget.hasVendorXTHeadMemPair())
14852 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
14854 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
14855 return {
Ptr->getOperand(0), C1->getZExtValue()};
14859 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
14882 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
14885 if (Base1 != Base2)
14889 bool Valid =
false;
14890 if (MemVT == MVT::i32) {
14892 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
14894 }
else if (MemVT == MVT::i64) {
14896 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
14930 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14938 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14948 EVT VT =
N->getValueType(0);
14951 MVT SrcVT = Src.getSimpleValueType();
14952 MVT SrcContainerVT = SrcVT;
14954 SDValue XVal = Src.getOperand(0);
14981 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14985 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14989 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
15002 if (VT != MVT::i32 && VT != XLenVT)
15032 EVT DstVT =
N->getValueType(0);
15033 if (DstVT != XLenVT)
15039 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15047 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15050 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
15059 if (SatVT == DstVT)
15061 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
15067 Src = Src.getOperand(0);
15088 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
15094 EVT VT =
N->getValueType(0);
15147 unsigned Offset =
N->isTargetStrictFPOpcode();
15154 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15156 V.getOperand(2) == VL) {
15158 V = V.getOperand(0);
15165 bool NegA = invertIfNegative(
A);
15166 bool NegB = invertIfNegative(
B);
15167 bool NegC = invertIfNegative(
C);
15170 if (!NegA && !NegB && !NegC)
15174 if (
N->isTargetStrictFPOpcode())
15176 {N->getOperand(0), A, B, C, Mask, VL});
15189 if (
N->getValueType(0).getVectorElementType() == MVT::f32 &&
15194 if (
N->isTargetStrictFPOpcode())
15204 EVT VT =
N->getValueType(0);
15209 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15211 uint64_t ShAmt =
N->getConstantOperandVal(1);
15219 cast<VTSDNode>(N0.
getOperand(1))->getVT().getSizeInBits();
15224 if (LShAmt < ExtSize) {
15237 if (ShAmt > 32 || VT != MVT::i64)
15253 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15266 !isa<ConstantSDNode>(U->getOperand(1)) ||
15267 U->getConstantOperandVal(1) > 32)
15322 if (!
Cond.hasOneUse())
15341 EVT VT =
Cond.getValueType();
15386 LHS =
LHS.getOperand(0);
15396 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15400 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15404 RHS =
LHS.getOperand(1);
15405 LHS =
LHS.getOperand(0);
15414 RHS =
LHS.getOperand(1);
15415 LHS =
LHS.getOperand(0);
15431 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15472 bool Commutative =
true;
15473 unsigned Opc = TrueVal.getOpcode();
15481 Commutative =
false;
15489 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15493 if (FalseVal == TrueVal.getOperand(0))
15495 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15500 EVT VT =
N->getValueType(0);
15502 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15508 assert(IdentityOperand &&
"No identity operand!");
15513 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15514 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15535 CountZeroes =
N->getOperand(2);
15536 ValOnZero =
N->getOperand(1);
15538 CountZeroes =
N->getOperand(1);
15539 ValOnZero =
N->getOperand(2);
15558 if (
Cond->getOperand(0) != CountZeroesArgument)
15574 CountZeroes, BitWidthMinusOne);
15584 EVT VT =
N->getValueType(0);
15585 EVT CondVT =
Cond.getValueType();
15593 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15599 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15620 SDValue TrueVal =
N->getOperand(1);
15621 SDValue FalseVal =
N->getOperand(2);
15636 EVT VT =
N->getValueType(0);
15643 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15658 if (
Op.isUndef()) {
15671 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15675 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15676 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15680 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15708 const unsigned InVecOpcode = InVec->
getOpcode();
15718 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15725 InVecLHS, InValLHS, EltNo);
15727 InVecRHS, InValRHS, EltNo);
15736 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15739 unsigned Elt = IndexC->getZExtValue();
15747 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15750 ConcatOp, InVal, NewIdx);
15754 ConcatOps[ConcatOpIdx] = ConcatOp;
15766 EVT VT =
N->getValueType(0);
15776 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
15778 !
SDValue(BaseLd, 0).hasOneUse())
15781 EVT BaseLdVT = BaseLd->getValueType(0);
15788 auto *Ld = dyn_cast<LoadSDNode>(
Op);
15789 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
15791 Ld->getValueType(0) != BaseLdVT)
15800 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
15802 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
15807 if (BIO1.equalBaseIndex(BIO2, DAG))
15812 SDValue P2 = Ld2->getBasePtr();
15818 return std::nullopt;
15822 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
15827 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
15828 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
15836 unsigned WideScalarBitWidth =
15849 auto [StrideVariant, MustNegateStride] = *BaseDiff;
15850 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
15851 ? std::get<SDValue>(StrideVariant)
15854 if (MustNegateStride)
15862 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
15863 ConstStride && ConstStride->getSExtValue() >= 0)
15867 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
15873 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
15877 WideVecVT,
DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
15892 if (
N->getValueType(0).isFixedLengthVector())
15895 SDValue Addend =
N->getOperand(0);
15899 SDValue AddPassthruOp =
N->getOperand(2);
15900 if (!AddPassthruOp.
isUndef())
15904 auto IsVWMulOpc = [](
unsigned Opc) {
15923 if (!MulPassthruOp.
isUndef())
15933 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
15934 }(
N, DAG, Subtarget);
15939 if (AddMask != MulMask || AddVL != MulVL)
15944 "Unexpected opcode after VWMACC_VL");
15946 "Unexpected opcode after VWMACC_VL!");
15948 "Unexpected opcode after VWMUL_VL!");
15950 "Unexpected opcode after VWMUL_VL!");
15953 EVT VT =
N->getValueType(0);
15969 const EVT IndexVT =
Index.getValueType();
15973 if (!isIndexTypeSigned(IndexType))
16005 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16008 if (
Index->getOperand(i)->isUndef())
16011 if (
C % ElementSize != 0)
16013 C =
C / ElementSize;
16017 ActiveLanes.
set(
C);
16019 return ActiveLanes.
all();
16037 if (NumElems % 2 != 0)
16041 const unsigned WiderElementSize = ElementSize * 2;
16042 if (WiderElementSize > ST.getELen()/8)
16045 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
16048 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16051 if (
Index->getOperand(i)->isUndef())
16057 if (
C % WiderElementSize != 0)
16062 if (
C !=
Last + ElementSize)
16079 (isa<RegisterSDNode>(VL) &&
16080 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16082 Mask.getOperand(0) != VL)
16085 auto IsTruncNode = [&](
SDValue V) {
16087 V.getOperand(1) == Mask && V.getOperand(2) == VL;
16094 while (IsTruncNode(
Op)) {
16095 if (!
Op.hasOneUse())
16097 Op =
Op.getOperand(0);
16132 MVT VT =
N->getSimpleValueType(0);
16137 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
16139 if (V.getOpcode() != Opc &&
16140 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
16141 V.getOperand(3) == Mask && V.getOperand(4) == VL))
16149 Op.getOperand(1).getValueType().isFixedLengthVector() &&
16151 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
16153 Op =
Op.getOperand(1).getOperand(0);
16156 return V.getOperand(0);
16159 Op.getOperand(2) == VL) {
16160 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
16162 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
16163 return V.getOperand(0);
16172 auto DetectUSatPattern = [&](
SDValue V) {
16197 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
16203 auto DetectSSatPattern = [&](
SDValue V) {
16205 unsigned NumSrcBits = V.getScalarValueSizeInBits();
16213 if (HiC == SignedMax && LoC == SignedMin)
16219 if (HiC == SignedMax && LoC == SignedMin)
16229 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
16231 Src = Src.getOperand(0);
16235 if ((Val = DetectUSatPattern(Src)))
16237 else if ((Val = DetectSSatPattern(Src)))
16247 Val = DAG.
getNode(ClipOpc,
DL, ValVT, Val, Mask, VL);
16248 }
while (ValVT != VT);
16262 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16273 switch (
N->getOpcode()) {
16293 APInt V =
C->getValueAPF().bitcastToAPInt();
16328 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16329 SimplifyDemandedLowBitsHelper(1, 5))
16337 if (SimplifyDemandedLowBitsHelper(0, 32))
16354 MVT VT =
N->getSimpleValueType(0);
16363 "Unexpected value type!");
16386 EVT VT =
N->getValueType(0);
16439 if (!
C || !
C->getValueAPF().isExactlyValue(+1.0))
16441 EVT VT =
N->getValueType(0);
16470 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16475 Src.getOperand(0));
16480 Src.getOperand(0), Src.getOperand(1));
16501 unsigned Opc =
N->getOpcode();
16516 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16526 N->getValueType(0), Val,
Cond.getOperand(0));
16537 SDValue FalseV =
N->getOperand(4);
16539 EVT VT =
N->getValueType(0);
16542 if (TrueV == FalseV)
16547 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16553 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16554 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16557 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16558 isInt<12>(TrueSImm - FalseSImm)) {
16574 {LHS, RHS, CC, TrueV, FalseV});
16641 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16654 EVT VT =
N->getValueType(0);
16678 const auto *MGN = cast<MaskedGatherSDNode>(
N);
16679 const EVT VT =
N->getValueType(0);
16681 SDValue ScaleOp = MGN->getScale();
16683 assert(!MGN->isIndexScaled() &&
16684 "Scaled gather/scatter should not be formed");
16689 N->getVTList(), MGN->getMemoryVT(),
DL,
16690 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16691 MGN->getBasePtr(), Index, ScaleOp},
16692 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16696 N->getVTList(), MGN->getMemoryVT(),
DL,
16697 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16698 MGN->getBasePtr(), Index, ScaleOp},
16699 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16705 if (std::optional<VIDSequence> SimpleVID =
16707 SimpleVID && SimpleVID->StepDenominator == 1) {
16708 const int64_t StepNumerator = SimpleVID->StepNumerator;
16709 const int64_t Addend = SimpleVID->Addend;
16716 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16726 MGN->getMask(), EVL, MGN->getMemOperand());
16728 StridedLoad, MGN->getPassThru(), EVL);
16738 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16740 MGN->getMemoryVT(), MGN->getMemOperand(),
16749 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16751 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16753 EVT IndexVT =
Index.getValueType()
16760 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16762 EltCnt.divideCoefficientBy(2));
16765 EltCnt.divideCoefficientBy(2));
16770 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16779 const auto *MSN = cast<MaskedScatterSDNode>(
N);
16781 SDValue ScaleOp = MSN->getScale();
16783 assert(!MSN->isIndexScaled() &&
16784 "Scaled gather/scatter should not be formed");
16789 N->getVTList(), MSN->getMemoryVT(),
DL,
16790 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16792 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16796 N->getVTList(), MSN->getMemoryVT(),
DL,
16797 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16799 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16801 EVT VT = MSN->getValue()->getValueType(0);
16803 if (!MSN->isTruncatingStore() &&
16807 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
16808 DAG.
getUNDEF(XLenVT), MSN->getMask(),
16809 MSN->getMemoryVT(), MSN->getMemOperand(),
16814 case ISD::VP_GATHER: {
16815 const auto *VPGN = cast<VPGatherSDNode>(
N);
16817 SDValue ScaleOp = VPGN->getScale();
16819 assert(!VPGN->isIndexScaled() &&
16820 "Scaled gather/scatter should not be formed");
16825 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16826 ScaleOp, VPGN->getMask(),
16827 VPGN->getVectorLength()},
16828 VPGN->getMemOperand(), IndexType);
16832 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16833 ScaleOp, VPGN->getMask(),
16834 VPGN->getVectorLength()},
16835 VPGN->getMemOperand(), IndexType);
16839 case ISD::VP_SCATTER: {
16840 const auto *VPSN = cast<VPScatterSDNode>(
N);
16842 SDValue ScaleOp = VPSN->getScale();
16844 assert(!VPSN->isIndexScaled() &&
16845 "Scaled gather/scatter should not be formed");
16850 {VPSN->getChain(), VPSN->getValue(),
16851 VPSN->getBasePtr(), Index, ScaleOp,
16852 VPSN->getMask(), VPSN->getVectorLength()},
16853 VPSN->getMemOperand(), IndexType);
16857 {VPSN->getChain(), VPSN->getValue(),
16858 VPSN->getBasePtr(), Index, ScaleOp,
16859 VPSN->getMask(), VPSN->getVectorLength()},
16860 VPSN->getMemOperand(), IndexType);
16874 EVT VT =
N->getValueType(0);
16877 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
16878 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
16896 EVT VT =
N->getValueType(0);
16900 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
16930 if (
N->getValueType(0).getVectorElementType() == MVT::f32 &&
16944 auto *Store = cast<StoreSDNode>(
N);
16945 SDValue Chain = Store->getChain();
16946 EVT MemVT = Store->getMemoryVT();
16947 SDValue Val = Store->getValue();
16950 bool IsScalarizable =
16952 Store->isSimple() &&
16982 NewVT, *Store->getMemOperand())) {
16984 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
16985 Store->getPointerInfo(), Store->getOriginalAlign(),
16986 Store->getMemOperand()->getFlags());
16994 if (
auto *L = dyn_cast<LoadSDNode>(Val);
16996 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
16998 L->getMemoryVT() == MemVT) {
17001 NewVT, *Store->getMemOperand()) &&
17003 NewVT, *L->getMemOperand())) {
17005 L->getPointerInfo(), L->getOriginalAlign(),
17006 L->getMemOperand()->getFlags());
17007 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
17008 Store->getPointerInfo(), Store->getOriginalAlign(),
17009 Store->getMemOperand()->getFlags());
17021 MVT VecVT = Src.getSimpleValueType();
17028 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
17031 Store->getMemOperand(), Store->getAddressingMode(),
17032 Store->isTruncatingStore(),
false);
17039 EVT VT =
N->getValueType(0);
17061 const MVT VT =
N->getSimpleValueType(0);
17062 SDValue Passthru =
N->getOperand(0);
17063 SDValue Scalar =
N->getOperand(1);
17072 const MVT VT =
N->getSimpleValueType(0);
17073 SDValue Passthru =
N->getOperand(0);
17074 SDValue Scalar =
N->getOperand(1);
17079 unsigned ScalarSize = Scalar.getValueSizeInBits();
17081 if (ScalarSize > EltWidth && Passthru.
isUndef())
17082 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
17089 (!Const || Const->isZero() ||
17090 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
17100 if (
N->getOperand(0).isUndef() &&
17103 Src.getOperand(0).getValueType().isScalableVector()) {
17104 EVT VT =
N->getValueType(0);
17105 EVT SrcVT = Src.getOperand(0).getValueType();
17109 return Src.getOperand(0);
17115 const MVT VT =
N->getSimpleValueType(0);
17116 SDValue Passthru =
N->getOperand(0);
17117 SDValue Scalar =
N->getOperand(1);
17121 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
17122 return Scalar.getOperand(0);
17131 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
17141 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
17149 MVT VecVT =
N->getOperand(0).getSimpleValueType();
17151 if (M1VT.
bitsLT(VecVT)) {
17162 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
17167 case Intrinsic::riscv_vcpop:
17168 case Intrinsic::riscv_vcpop_mask:
17169 case Intrinsic::riscv_vfirst:
17170 case Intrinsic::riscv_vfirst_mask: {
17172 if (IntNo == Intrinsic::riscv_vcpop_mask ||
17173 IntNo == Intrinsic::riscv_vfirst_mask)
17174 VL =
N->getOperand(3);
17179 EVT VT =
N->getValueType(0);
17180 if (IntNo == Intrinsic::riscv_vfirst ||
17181 IntNo == Intrinsic::riscv_vfirst_mask)
17190 EVT VT =
N->getValueType(0);
17194 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17213 EVT XVT,
unsigned KeptBits)
const {
17218 if (XVT != MVT::i32 && XVT != MVT::i64)
17222 if (KeptBits == 32 || KeptBits == 64)
17226 return Subtarget.hasStdExtZbb() &&
17227 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17235 "Expected shift op");
17246 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
17247 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17249 const APInt &C1Int = C1->getAPIntValue();
17250 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17276 if (C1Cost < ShiftedC1Cost)
17290 EVT VT =
Op.getValueType();
17294 unsigned Opcode =
Op.getOpcode();
17302 const APInt &Mask =
C->getAPIntValue();
17311 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17312 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17314 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17315 if (NewMask == Mask)
17320 Op.getOperand(0), NewC);
17333 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17334 if (IsLegalMask(NewMask))
17335 return UseMask(NewMask);
17338 if (VT == MVT::i64) {
17340 if (IsLegalMask(NewMask))
17341 return UseMask(NewMask);
17356 APInt NewMask = ShrunkMask;
17357 if (MinSignedBits <= 12)
17359 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17365 assert(IsLegalMask(NewMask));
17366 return UseMask(NewMask);
17370 static const uint64_t GREVMasks[] = {
17371 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17372 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17374 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17375 unsigned Shift = 1 << Stage;
17376 if (ShAmt & Shift) {
17378 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17390 const APInt &DemandedElts,
17392 unsigned Depth)
const {
17394 unsigned Opc =
Op.getOpcode();
17399 "Should use MaskedValueIsZero if you don't know whether Op"
17400 " is a target node!");
17483 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17486 if (MaxVLenB == MinVLenB)
17503 case Intrinsic::riscv_vsetvli:
17504 case Intrinsic::riscv_vsetvlimax: {
17505 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17506 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17512 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17515 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17516 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17518 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17531 unsigned Depth)
const {
17532 switch (
Op.getOpcode()) {
17538 if (Tmp == 1)
return 1;
17541 return std::min(Tmp, Tmp2);
17553 if (Tmp < 33)
return 1;
17578 unsigned XLen = Subtarget.
getXLen();
17579 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17580 if (EltBits <= XLen)
17581 return XLen - EltBits + 1;
17585 unsigned IntNo =
Op.getConstantOperandVal(1);
17589 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17590 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17591 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17592 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17593 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17594 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17595 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17596 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17597 case Intrinsic::riscv_masked_cmpxchg_i64:
17605 assert(Subtarget.hasStdExtA());
17620 switch (
Op.getOpcode()) {
17626 return !
Op.getValueType().isInteger();
17634 assert(Ld &&
"Unexpected null LoadSDNode");
17642 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17643 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17644 CNode->getOffset() != 0)
17652 auto *CNode = GetSupportedConstantPool(
Ptr);
17653 if (!CNode || CNode->getTargetFlags() != 0)
17656 return CNode->getConstVal();
17664 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17665 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17671 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17674 return CNodeLo->getConstVal();
17679 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17711 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17714 int64_t LoCounter =
MI.getOperand(2).getImm();
17715 int64_t HiCounter =
MI.getOperand(3).getImm();
17725 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17737 MI.eraseFromParent();
17745 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
17753 Register SrcReg =
MI.getOperand(2).getReg();
17773 MI.eraseFromParent();
17780 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
17781 "Unexpected instruction");
17787 Register DstReg =
MI.getOperand(0).getReg();
17810 MI.eraseFromParent();
17815 switch (
MI.getOpcode()) {
17818 case RISCV::Select_GPR_Using_CC_GPR:
17819 case RISCV::Select_GPR_Using_CC_Imm:
17820 case RISCV::Select_FPR16_Using_CC_GPR:
17821 case RISCV::Select_FPR16INX_Using_CC_GPR:
17822 case RISCV::Select_FPR32_Using_CC_GPR:
17823 case RISCV::Select_FPR32INX_Using_CC_GPR:
17824 case RISCV::Select_FPR64_Using_CC_GPR:
17825 case RISCV::Select_FPR64INX_Using_CC_GPR:
17826 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
17832 unsigned RelOpcode,
unsigned EqOpcode,
17835 Register DstReg =
MI.getOperand(0).getReg();
17836 Register Src1Reg =
MI.getOperand(1).getReg();
17837 Register Src2Reg =
MI.getOperand(2).getReg();
17839 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17863 MI.eraseFromParent();
17914 F->insert(It, FirstMBB);
17915 F->insert(It, SecondMBB);
17916 F->insert(It, SinkMBB);
17965 First.eraseFromParent();
18004 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
18005 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
18006 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
18007 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
18008 Next->getOperand(5).isKill())
18013 if (
MI.getOperand(2).isReg())
18014 RHS =
MI.getOperand(2).getReg();
18019 SelectDests.
insert(
MI.getOperand(0).getReg());
18023 SequenceMBBI != E; ++SequenceMBBI) {
18024 if (SequenceMBBI->isDebugInstr())
18027 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
18028 !SequenceMBBI->getOperand(2).isReg() ||
18029 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
18030 SequenceMBBI->getOperand(3).getImm() !=
CC ||
18031 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
18032 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
18034 LastSelectPseudo = &*SequenceMBBI;
18036 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
18039 if (SequenceMBBI->hasUnmodeledSideEffects() ||
18040 SequenceMBBI->mayLoadOrStore() ||
18041 SequenceMBBI->usesCustomInsertionHook())
18044 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
18059 F->insert(
I, IfFalseMBB);
18060 F->insert(
I, TailMBB);
18063 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
18069 TailMBB->
push_back(DebugInstr->removeFromParent());
18073 TailMBB->
splice(TailMBB->
end(), HeadMBB,
18083 if (
MI.getOperand(2).isImm())
18086 .
addImm(
MI.getOperand(2).getImm())
18098 auto SelectMBBI =
MI.getIterator();
18099 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
18100 auto InsertionPoint = TailMBB->
begin();
18101 while (SelectMBBI != SelectEnd) {
18102 auto Next = std::next(SelectMBBI);
18105 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
18106 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
18107 .
addReg(SelectMBBI->getOperand(4).getReg())
18109 .
addReg(SelectMBBI->getOperand(5).getReg())
18124 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
18125 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
18127 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
18128 assert(
Masked &&
"Could not find masked instruction for LMUL and SEW pair");
18134 unsigned CVTXOpc) {
18140 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18153 .
add(
MI.getOperand(1))
18154 .
add(
MI.getOperand(2))
18155 .
add(
MI.getOperand(3))
18157 .
add(
MI.getOperand(4))
18158 .
add(
MI.getOperand(5))
18159 .
add(
MI.getOperand(6))
18174 .
add(
MI.getOperand(0))
18175 .
add(
MI.getOperand(1))
18177 .
add(
MI.getOperand(3))
18179 .
add(
MI.getOperand(4))
18180 .
add(
MI.getOperand(5))
18181 .
add(
MI.getOperand(6))
18191 MI.eraseFromParent();
18197 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
18199 switch (
MI.getOpcode()) {
18202 case RISCV::PseudoFROUND_H:
18203 CmpOpc = RISCV::FLT_H;
18204 F2IOpc = RISCV::FCVT_W_H;
18205 I2FOpc = RISCV::FCVT_H_W;
18206 FSGNJOpc = RISCV::FSGNJ_H;
18207 FSGNJXOpc = RISCV::FSGNJX_H;
18208 RC = &RISCV::FPR16RegClass;
18210 case RISCV::PseudoFROUND_H_INX:
18211 CmpOpc = RISCV::FLT_H_INX;
18212 F2IOpc = RISCV::FCVT_W_H_INX;
18213 I2FOpc = RISCV::FCVT_H_W_INX;
18214 FSGNJOpc = RISCV::FSGNJ_H_INX;
18215 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18216 RC = &RISCV::GPRF16RegClass;
18218 case RISCV::PseudoFROUND_S:
18219 CmpOpc = RISCV::FLT_S;
18220 F2IOpc = RISCV::FCVT_W_S;
18221 I2FOpc = RISCV::FCVT_S_W;
18222 FSGNJOpc = RISCV::FSGNJ_S;
18223 FSGNJXOpc = RISCV::FSGNJX_S;
18224 RC = &RISCV::FPR32RegClass;
18226 case RISCV::PseudoFROUND_S_INX:
18227 CmpOpc = RISCV::FLT_S_INX;
18228 F2IOpc = RISCV::FCVT_W_S_INX;
18229 I2FOpc = RISCV::FCVT_S_W_INX;
18230 FSGNJOpc = RISCV::FSGNJ_S_INX;
18231 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18232 RC = &RISCV::GPRF32RegClass;
18234 case RISCV::PseudoFROUND_D:
18236 CmpOpc = RISCV::FLT_D;
18237 F2IOpc = RISCV::FCVT_L_D;
18238 I2FOpc = RISCV::FCVT_D_L;
18239 FSGNJOpc = RISCV::FSGNJ_D;
18240 FSGNJXOpc = RISCV::FSGNJX_D;
18241 RC = &RISCV::FPR64RegClass;
18243 case RISCV::PseudoFROUND_D_INX:
18245 CmpOpc = RISCV::FLT_D_INX;
18246 F2IOpc = RISCV::FCVT_L_D_INX;
18247 I2FOpc = RISCV::FCVT_D_L_INX;
18248 FSGNJOpc = RISCV::FSGNJ_D_INX;
18249 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18250 RC = &RISCV::GPRRegClass;
18262 F->insert(
I, CvtMBB);
18263 F->insert(
I, DoneMBB);
18274 Register DstReg =
MI.getOperand(0).getReg();
18275 Register SrcReg =
MI.getOperand(1).getReg();
18276 Register MaxReg =
MI.getOperand(2).getReg();
18277 int64_t FRM =
MI.getOperand(3).getImm();
18282 Register FabsReg =
MRI.createVirtualRegister(RC);
18286 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18301 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18323 MI.eraseFromParent();
18330 switch (
MI.getOpcode()) {
18333 case RISCV::ReadCounterWide:
18335 "ReadCounterWide is only to be used on riscv32");
18337 case RISCV::Select_GPR_Using_CC_GPR:
18338 case RISCV::Select_GPR_Using_CC_Imm:
18339 case RISCV::Select_FPR16_Using_CC_GPR:
18340 case RISCV::Select_FPR16INX_Using_CC_GPR:
18341 case RISCV::Select_FPR32_Using_CC_GPR:
18342 case RISCV::Select_FPR32INX_Using_CC_GPR:
18343 case RISCV::Select_FPR64_Using_CC_GPR:
18344 case RISCV::Select_FPR64INX_Using_CC_GPR:
18345 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18347 case RISCV::BuildPairF64Pseudo:
18349 case RISCV::SplitF64Pseudo:
18351 case RISCV::PseudoQuietFLE_H:
18353 case RISCV::PseudoQuietFLE_H_INX:
18354 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18355 case RISCV::PseudoQuietFLT_H:
18357 case RISCV::PseudoQuietFLT_H_INX:
18358 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18359 case RISCV::PseudoQuietFLE_S:
18361 case RISCV::PseudoQuietFLE_S_INX:
18362 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18363 case RISCV::PseudoQuietFLT_S:
18365 case RISCV::PseudoQuietFLT_S_INX:
18366 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18367 case RISCV::PseudoQuietFLE_D:
18369 case RISCV::PseudoQuietFLE_D_INX:
18370 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18371 case RISCV::PseudoQuietFLE_D_IN32X:
18374 case RISCV::PseudoQuietFLT_D:
18376 case RISCV::PseudoQuietFLT_D_INX:
18377 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18378 case RISCV::PseudoQuietFLT_D_IN32X:
18382 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18384 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18386 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18388 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18390 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18392 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18394 case RISCV::PseudoFROUND_H:
18395 case RISCV::PseudoFROUND_H_INX:
18396 case RISCV::PseudoFROUND_S:
18397 case RISCV::PseudoFROUND_S_INX:
18398 case RISCV::PseudoFROUND_D:
18399 case RISCV::PseudoFROUND_D_INX:
18400 case RISCV::PseudoFROUND_D_IN32X:
18402 case TargetOpcode::STATEPOINT:
18408 MI.addOperand(*
MI.getMF(),
18414 case TargetOpcode::STACKMAP:
18415 case TargetOpcode::PATCHPOINT:
18418 "supported on 64-bit targets");
18436 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18468 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18469 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18472 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18473 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18476 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18477 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18481 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18482 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18483 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18485 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18486 RISCV::V20M2, RISCV::V22M2};
18494 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18495 RISCV::X13, RISCV::X14, RISCV::X15,
18496 RISCV::X16, RISCV::X17};
18498 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18499 RISCV::X13, RISCV::X14, RISCV::X15};
18511 static const MCPhysReg FastCCIGPRs[] = {
18512 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15,
18513 RISCV::X16, RISCV::X17, RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31};
18516 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18517 RISCV::X13, RISCV::X14, RISCV::X15};
18531 unsigned XLenInBytes = XLen / 8;
18544 Align StackAlign(XLenInBytes);
18545 if (!
EABI || XLen != 32)
18577 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18578 assert(XLen == 32 || XLen == 64);
18579 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18583 if (ArgFlags.
isNest()) {
18584 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18592 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18597 bool UseGPRForF16_F32 =
true;
18600 bool UseGPRForF64 =
true;
18612 UseGPRForF16_F32 = !IsFixed;
18616 UseGPRForF16_F32 = !IsFixed;
18617 UseGPRForF64 = !IsFixed;
18623 UseGPRForF16_F32 =
true;
18624 UseGPRForF64 =
true;
18631 if (UseGPRForF16_F32 &&
18632 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18635 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18652 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18654 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18658 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18667 "PendingLocs and PendingArgFlags out of sync");
18671 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18672 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18720 PendingLocs.
size() <= 2) {
18721 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18726 PendingLocs.
clear();
18727 PendingArgFlags.
clear();
18729 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18735 unsigned StoreSizeBytes = XLen / 8;
18738 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18740 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18742 else if (ValVT == MVT::f64 && !UseGPRForF64)
18778 if (!PendingLocs.
empty()) {
18780 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
18782 for (
auto &It : PendingLocs) {
18784 It.convertToReg(Reg);
18789 PendingLocs.clear();
18790 PendingArgFlags.
clear();
18794 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
18796 "Expected an XLenVT or vector types at this stage");
18814template <
typename ArgTy>
18816 for (
const auto &ArgIdx :
enumerate(Args)) {
18817 MVT ArgVT = ArgIdx.value().VT;
18819 return ArgIdx.index();
18821 return std::nullopt;
18824void RISCVTargetLowering::analyzeInputArgs(
18827 RISCVCCAssignFn Fn)
const {
18828 unsigned NumArgs =
Ins.size();
18841 for (
unsigned i = 0; i != NumArgs; ++i) {
18845 Type *ArgTy =
nullptr;
18847 ArgTy = FType->getReturnType();
18848 else if (Ins[i].isOrigArg())
18849 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
18853 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
18855 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
18862void RISCVTargetLowering::analyzeOutputArgs(
18865 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
18866 unsigned NumArgs = Outs.
size();
18876 for (
unsigned i = 0; i != NumArgs; i++) {
18877 MVT ArgVT = Outs[i].VT;
18879 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
18883 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
18885 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
18934 if (In.isOrigArg()) {
18939 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
18940 (
BitWidth < 32 && In.Flags.isZExt())) {
18969 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
19010 ExtType,
DL, LocVT, Chain, FIN,
19027 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19040 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19050 unsigned ValNo,
MVT ValVT,
MVT LocVT,
19053 bool IsFixed,
bool IsRet,
Type *OrigTy,
19056 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19065 if (LocVT == MVT::f16 &&
19066 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
19068 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
19069 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
19070 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
19071 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
19072 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
19078 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19080 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
19081 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
19082 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
19083 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
19084 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19090 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19092 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
19093 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
19094 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
19095 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
19096 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19103 if ((LocVT == MVT::f16 &&
19104 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
19105 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19106 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
19107 Subtarget.hasStdExtZdinx())) {
19114 if (LocVT == MVT::f16) {
19120 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
19126 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
19134 if (AllocatedVReg) {
19169 if (ArgFlags.
isNest()) {
19171 "Attribute 'nest' is not supported in GHC calling convention");
19175 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
19176 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
19178 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19190 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19193 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
19194 RISCV::F18_F, RISCV::F19_F,
19195 RISCV::F20_F, RISCV::F21_F};
19196 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19202 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19205 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
19206 RISCV::F24_D, RISCV::F25_D,
19207 RISCV::F26_D, RISCV::F27_D};
19208 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19214 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19215 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
19235 switch (CallConv) {
19245 if (Subtarget.hasStdExtE())
19249 "(Zdinx/D) instruction set extensions");
19253 if (Func.hasFnAttribute(
"interrupt")) {
19254 if (!Func.arg_empty())
19256 "Functions with the interrupt attribute cannot have arguments!");
19261 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19263 "Function interrupt attribute argument not supported!");
19268 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19270 std::vector<SDValue> OutChains;
19279 analyzeInputArgs(MF, CCInfo, Ins,
false,
19283 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19304 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19305 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19307 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19309 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19338 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19343 if (VarArgsSaveSize == 0) {
19347 int VaArgOffset = -VarArgsSaveSize;
19355 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19356 VarArgsSaveSize += XLenInBytes;
19363 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19368 Chain,
DL, ArgValue, FIN,
19370 OutChains.push_back(Store);
19384 if (!OutChains.empty()) {
19385 OutChains.push_back(Chain);
19395bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19399 auto CalleeCC = CLI.CallConv;
19400 auto &Outs = CLI.Outs;
19402 auto CallerCC = Caller.getCallingConv();
19409 if (Caller.hasFnAttribute(
"interrupt"))
19424 for (
auto &VA : ArgLocs)
19430 auto IsCallerStructRet = Caller.hasStructRetAttr();
19431 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19432 if (IsCallerStructRet || IsCalleeStructRet)
19437 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19438 if (CalleeCC != CallerCC) {
19439 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19440 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19447 for (
auto &Arg : Outs)
19448 if (Arg.Flags.isByVal())
19483 if (Subtarget.hasStdExtE())
19487 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19493 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19499 "site marked musttail");
19506 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19508 if (!Flags.isByVal())
19512 unsigned Size = Flags.getByValSize();
19513 Align Alignment = Flags.getNonZeroByValAlign();
19520 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19522 false,
nullptr, IsTailCall,
19534 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19537 SDValue ArgValue = OutVals[OutIdx];
19557 if (!StackPtr.getNode())
19568 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19586 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19587 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19593 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19594 SDValue PartValue = OutVals[OutIdx + 1];
19595 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19607 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19609 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19611 for (
const auto &Part : Parts) {
19612 SDValue PartValue = Part.first;
19613 SDValue PartOffset = Part.second;
19620 ArgValue = SpillSlot;
19626 if (Flags.isByVal())
19627 ArgValue = ByValArgs[j++];
19634 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19635 "for passing parameters");
19638 if (!StackPtr.getNode())
19651 if (!MemOpChains.
empty())
19657 for (
auto &Reg : RegsToPass) {
19658 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19665 validateCCReservedRegs(RegsToPass, MF);
19670 "Return address register required, but has been reserved."});
19689 for (
auto &Reg : RegsToPass)
19695 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19696 assert(Mask &&
"Missing call preserved mask for calling convention");
19705 "Unexpected CFI type for a direct call");
19735 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19736 auto &VA = RVLocs[i];
19744 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19745 assert(VA.needsCustom());
19766 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
19770 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19771 MVT VT = Outs[i].VT;
19775 ArgFlags, CCInfo,
true,
true,
19776 nullptr, *
this, Dispatcher))
19808 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19809 SDValue Val = OutVals[OutIdx];
19818 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19822 Register RegHi = RVLocs[++i].getLocReg();
19828 "Return value register required, but has been reserved."});
19844 "Return value register required, but has been reserved."});
19866 if (Func.hasFnAttribute(
"interrupt")) {
19867 if (!Func.getReturnType()->isVoidTy())
19869 "Functions with the interrupt attribute must have void return type!");
19875 if (Kind ==
"supervisor")
19881 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19884void RISCVTargetLowering::validateCCReservedRegs(
19885 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19894 F,
"Argument register required, but has been reserved."});
19900 if (
N->getNumValues() != 1)
19902 if (!
N->hasNUsesOfValue(1, 0))
19905 SDNode *Copy = *
N->use_begin();
19919 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
19923 bool HasRet =
false;
19924 for (
SDNode *Node : Copy->uses()) {
19932 Chain = Copy->getOperand(0);
19941#define NODE_NAME_CASE(NODE) \
19942 case RISCVISD::NODE: \
19943 return "RISCVISD::" #NODE;
20202#undef NODE_NAME_CASE
20209 if (Constraint.
size() == 1) {
20210 switch (Constraint[0]) {
20226 if (Constraint ==
"vr" || Constraint ==
"vm")
20232std::pair<unsigned, const TargetRegisterClass *>
20238 if (Constraint.
size() == 1) {
20239 switch (Constraint[0]) {
20244 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20245 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20246 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20247 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20248 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20249 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20250 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20252 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20253 return std::make_pair(0U, &RISCV::FPR16RegClass);
20254 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20255 return std::make_pair(0U, &RISCV::FPR32RegClass);
20256 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20257 return std::make_pair(0U, &RISCV::FPR64RegClass);
20262 }
else if (Constraint ==
"vr") {
20263 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
20264 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20266 return std::make_pair(0U, RC);
20268 }
else if (Constraint ==
"vm") {
20269 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20270 return std::make_pair(0U, &RISCV::VMV0RegClass);
20278 .
Case(
"{zero}", RISCV::X0)
20279 .
Case(
"{ra}", RISCV::X1)
20280 .
Case(
"{sp}", RISCV::X2)
20281 .
Case(
"{gp}", RISCV::X3)
20282 .
Case(
"{tp}", RISCV::X4)
20283 .
Case(
"{t0}", RISCV::X5)
20284 .
Case(
"{t1}", RISCV::X6)
20285 .
Case(
"{t2}", RISCV::X7)
20286 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20287 .
Case(
"{s1}", RISCV::X9)
20288 .
Case(
"{a0}", RISCV::X10)
20289 .
Case(
"{a1}", RISCV::X11)
20290 .
Case(
"{a2}", RISCV::X12)
20291 .
Case(
"{a3}", RISCV::X13)
20292 .
Case(
"{a4}", RISCV::X14)
20293 .
Case(
"{a5}", RISCV::X15)
20294 .
Case(
"{a6}", RISCV::X16)
20295 .
Case(
"{a7}", RISCV::X17)
20296 .
Case(
"{s2}", RISCV::X18)
20297 .
Case(
"{s3}", RISCV::X19)
20298 .
Case(
"{s4}", RISCV::X20)
20299 .
Case(
"{s5}", RISCV::X21)
20300 .
Case(
"{s6}", RISCV::X22)
20301 .
Case(
"{s7}", RISCV::X23)
20302 .
Case(
"{s8}", RISCV::X24)
20303 .
Case(
"{s9}", RISCV::X25)
20304 .
Case(
"{s10}", RISCV::X26)
20305 .
Case(
"{s11}", RISCV::X27)
20306 .
Case(
"{t3}", RISCV::X28)
20307 .
Case(
"{t4}", RISCV::X29)
20308 .
Case(
"{t5}", RISCV::X30)
20309 .
Case(
"{t6}", RISCV::X31)
20311 if (XRegFromAlias != RISCV::NoRegister)
20312 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20321 if (Subtarget.hasStdExtF()) {
20323 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20324 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20325 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20326 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20327 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20328 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20329 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20330 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20331 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20332 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20333 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20334 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20335 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20336 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20337 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20338 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20339 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20340 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20341 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20342 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20343 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20344 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20345 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20346 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20347 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20348 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20349 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20350 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20351 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20352 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20353 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20354 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20356 if (FReg != RISCV::NoRegister) {
20357 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20358 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20359 unsigned RegNo = FReg - RISCV::F0_F;
20360 unsigned DReg = RISCV::F0_D + RegNo;
20361 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20363 if (VT == MVT::f32 || VT == MVT::Other)
20364 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20365 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20366 unsigned RegNo = FReg - RISCV::F0_F;
20367 unsigned HReg = RISCV::F0_H + RegNo;
20368 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20375 .
Case(
"{v0}", RISCV::V0)
20376 .
Case(
"{v1}", RISCV::V1)
20377 .
Case(
"{v2}", RISCV::V2)
20378 .
Case(
"{v3}", RISCV::V3)
20379 .
Case(
"{v4}", RISCV::V4)
20380 .
Case(
"{v5}", RISCV::V5)
20381 .
Case(
"{v6}", RISCV::V6)
20382 .
Case(
"{v7}", RISCV::V7)
20383 .
Case(
"{v8}", RISCV::V8)
20384 .
Case(
"{v9}", RISCV::V9)
20385 .
Case(
"{v10}", RISCV::V10)
20386 .
Case(
"{v11}", RISCV::V11)
20387 .
Case(
"{v12}", RISCV::V12)
20388 .
Case(
"{v13}", RISCV::V13)
20389 .
Case(
"{v14}", RISCV::V14)
20390 .
Case(
"{v15}", RISCV::V15)
20391 .
Case(
"{v16}", RISCV::V16)
20392 .
Case(
"{v17}", RISCV::V17)
20393 .
Case(
"{v18}", RISCV::V18)
20394 .
Case(
"{v19}", RISCV::V19)
20395 .
Case(
"{v20}", RISCV::V20)
20396 .
Case(
"{v21}", RISCV::V21)
20397 .
Case(
"{v22}", RISCV::V22)
20398 .
Case(
"{v23}", RISCV::V23)
20399 .
Case(
"{v24}", RISCV::V24)
20400 .
Case(
"{v25}", RISCV::V25)
20401 .
Case(
"{v26}", RISCV::V26)
20402 .
Case(
"{v27}", RISCV::V27)
20403 .
Case(
"{v28}", RISCV::V28)
20404 .
Case(
"{v29}", RISCV::V29)
20405 .
Case(
"{v30}", RISCV::V30)
20406 .
Case(
"{v31}", RISCV::V31)
20408 if (VReg != RISCV::NoRegister) {
20409 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20410 return std::make_pair(VReg, &RISCV::VMRegClass);
20411 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20412 return std::make_pair(VReg, &RISCV::VRRegClass);
20413 for (
const auto *RC :
20414 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20415 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20416 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20417 return std::make_pair(VReg, RC);
20423 std::pair<Register, const TargetRegisterClass *> Res =
20429 if (Res.second == &RISCV::GPRF16RegClass ||
20430 Res.second == &RISCV::GPRF32RegClass ||
20431 Res.second == &RISCV::GPRPairRegClass)
20432 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20440 if (ConstraintCode.
size() == 1) {
20441 switch (ConstraintCode[0]) {
20456 if (Constraint.
size() == 1) {
20457 switch (Constraint[0]) {
20460 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20462 if (isInt<12>(CVal))
20475 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20477 if (isUInt<5>(CVal))
20495 if (Subtarget.hasStdExtZtso()) {
20511 if (Subtarget.hasStdExtZtso()) {
20519 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Inst) &&
20536 if (Subtarget.hasForcedAtomics())
20541 if (Subtarget.hasStdExtZacas() &&
20542 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20548 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20561 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20563 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20565 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20567 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20569 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20571 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20573 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20575 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20584 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20586 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20588 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20590 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20592 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20594 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20596 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20598 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20624 unsigned XLen = Subtarget.
getXLen();
20648 unsigned ValWidth =
20653 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20656 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20668 if (Subtarget.hasForcedAtomics())
20672 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20681 unsigned XLen = Subtarget.
getXLen();
20683 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20688 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20694 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20701 EVT DataVT)
const {
20717 return Subtarget.hasStdExtZfhmin();
20719 return Subtarget.hasStdExtF();
20721 return Subtarget.hasStdExtD();
20753 "RVVBitsPerBlock changed, audit needed");
20762 if (!Subtarget.hasVendorXTHeadMemIdx())
20768 Base =
Op->getOperand(0);
20770 int64_t RHSC =
RHS->getSExtValue();
20776 bool isLegalIndexedOffset =
false;
20777 for (
unsigned i = 0; i < 4; i++)
20778 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20779 isLegalIndexedOffset =
true;
20783 if (!isLegalIndexedOffset)
20800 VT = LD->getMemoryVT();
20801 Ptr = LD->getBasePtr();
20802 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20803 VT = ST->getMemoryVT();
20804 Ptr = ST->getBasePtr();
20820 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
20825 Base = LS->getBasePtr();
20829 if (
Base ==
Op->getOperand(0))
20831 else if (
Base ==
Op->getOperand(1))
20843 VT = LD->getMemoryVT();
20844 Ptr = LD->getBasePtr();
20845 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20846 VT = ST->getMemoryVT();
20847 Ptr = ST->getBasePtr();
20890 const Constant *PersonalityFn)
const {
20895 const Constant *PersonalityFn)
const {
20924 const bool HasZmmul = Subtarget.hasStdExtZmmul();
20928 auto *ConstNode = cast<ConstantSDNode>(
C);
20929 const APInt &Imm = ConstNode->getAPIntValue();
20932 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
20933 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
20937 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
20938 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
20939 (Imm - 8).isPowerOf2()))
20944 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
20945 ConstNode->hasOneUse()) {
20946 APInt ImmS = Imm.ashr(Imm.countr_zero());
20947 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
20948 (1 - ImmS).isPowerOf2())
20971 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
20980 unsigned *
Fast)
const {
20983 *
Fast = Subtarget.enableUnalignedScalarMem();
20984 return Subtarget.enableUnalignedScalarMem();
21000 *
Fast = Subtarget.enableUnalignedVectorMem();
21001 return Subtarget.enableUnalignedVectorMem();
21010 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
21022 if (
Op.size() < MinVLenInBytes)
21037 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
21041 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
21043 if (
Op.isFixedDstAlign())
21044 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
21046 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
21054 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
21055 bool IsABIRegCopy =
CC.has_value();
21057 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21058 PartVT == MVT::f32) {
21076 if (PartVTBitSize % ValueVTBitSize == 0) {
21077 assert(PartVTBitSize >= ValueVTBitSize);
21084 if (ValueEltVT != PartEltVT) {
21085 if (PartVTBitSize > ValueVTBitSize) {
21087 assert(Count != 0 &&
"The number of element should not be zero.");
21088 EVT SameEltTypeVT =
21109 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
21110 bool IsABIRegCopy =
CC.has_value();
21111 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21112 PartVT == MVT::f32) {
21129 if (PartVTBitSize % ValueVTBitSize == 0) {
21130 assert(PartVTBitSize >= ValueVTBitSize);
21131 EVT SameEltTypeVT = ValueVT;
21138 if (ValueEltVT != PartEltVT) {
21140 assert(Count != 0 &&
"The number of element should not be zero.");
21157 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21164 unsigned Opc =
N->getOpcode();
21194 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21208 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21213 if (FVTy->getNumElements() < 2)
21228 return Factor * LMUL <= 8;
21232 Align Alignment)
const {
21244 if (!Subtarget.enableUnalignedVectorMem() &&
21252 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21253 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21254 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21255 Intrinsic::riscv_seg8_load};
21274 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21284 {VTy, LI->getPointerOperandType(), XLenTy});
21286 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21291 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21293 Shuffles[i]->replaceAllUsesWith(SubVec);
21300 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21301 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21302 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21303 Intrinsic::riscv_seg8_store};
21323 unsigned Factor)
const {
21325 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21328 ShuffleVTy->getNumElements() / Factor);
21330 SI->getPointerAddressSpace(),
21331 SI->getDataLayout()))
21338 {VTy, SI->getPointerOperandType(), XLenTy});
21343 for (
unsigned i = 0; i < Factor; i++) {
21352 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21353 Ops.
append({SI->getPointerOperand(), VL});
21370 unsigned Factor = 2;
21385 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21388 {ResVTy, LI->getPointerOperandType(), XLenTy});
21389 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21392 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21393 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21394 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21395 Intrinsic::riscv_vlseg8};
21418 if (
II->getIntrinsicID() != Intrinsic::vector_interleave2)
21421 unsigned Factor = 2;
21424 VectorType *InVTy = cast<VectorType>(
II->getOperand(0)->getType());
21427 SI->getPointerAddressSpace(),
21428 SI->getDataLayout()))
21435 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21438 {InVTy, SI->getPointerOperandType(), XLenTy});
21439 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21442 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21443 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21444 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21445 Intrinsic::riscv_vsseg8};
21452 Builder.
CreateCall(VssegNFunc, {
II->getOperand(0),
II->getOperand(1),
21453 SI->getPointerOperand(), VL});
21463 "Invalid call instruction for a KCFI check");
21465 MBBI->getOpcode()));
21468 Target.setIsRenamable(
false);
21476#define GET_REGISTER_MATCHER
21477#include "RISCVGenAsmMatcher.inc"
21483 if (Reg == RISCV::NoRegister)
21485 if (Reg == RISCV::NoRegister)
21497 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21499 if (NontemporalInfo ==
nullptr)
21507 int NontemporalLevel = 5;
21508 const MDNode *RISCVNontemporalInfo =
21509 I.getMetadata(
"riscv-nontemporal-domain");
21510 if (RISCVNontemporalInfo !=
nullptr)
21513 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21517 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21518 "RISC-V target doesn't support this non-temporal domain.");
21520 NontemporalLevel -= 2;
21522 if (NontemporalLevel & 0b1)
21524 if (NontemporalLevel & 0b10)
21537 return TargetFlags;
21547 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21550 return Subtarget.hasStdExtZbb() &&
21563 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21564 Op == Instruction::And ||
Op == Instruction::Or ||
21565 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21566 Op == Instruction::ShuffleVector ||
Op == Instruction::Load ||
21567 Op == Instruction::Freeze ||
Op == Instruction::Store)
21575 !isa<ReturnInst>(&Inst))
21578 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21579 if (AI->getAllocatedType()->isScalableTy())
21587RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21595 if (!Subtarget.hasShortForwardBranchOpt())
21597 EVT VT =
N->getValueType(0);
21598 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21602 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21607bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21608 EVT VT,
const APInt &AndMask)
const {
21609 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21610 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21614unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21619template <
typename Arg>
21620void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21623 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21625 auto It = ArgList.
begin();
21626 MVT FirstArgRegType = It->VT;
21629 if (It == ArgList.
end() || It->Flags.isSplit())
21641 for (; It != ArgList.
end(); ++It)
21642 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21648 if (isHomogeneousScalableVectorType(ArgList)) {
21650 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21653 bool FirstVMaskAssigned =
false;
21654 for (
const auto &OutArg : ArgList) {
21655 MVT RegisterVT = OutArg.VT;
21665 RVVArgInfos.push_back({1, RegisterVT,
true});
21666 FirstVMaskAssigned =
true;
21670 RVVArgInfos.push_back({1, RegisterVT,
false});
21677void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21682 bool FirstVMaskAssigned =
false;
21683 for (
Type *Ty : TypeList) {
21693 RVVArgInfos.push_back(
21699 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21714 if (!FirstVMaskAssigned &&
21716 RVVArgInfos.push_back({1, RegisterVT,
true});
21717 FirstVMaskAssigned =
true;
21721 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21727void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21728 unsigned StartReg) {
21729 assert((StartReg % LMul) == 0 &&
21730 "Start register number should be multiple of lmul");
21749 for (
unsigned i = 0; i < NF; ++i)
21751 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
21753 AllocatedPhysRegs.push_back(
MCPhysReg());
21759void RVVArgDispatcher::compute() {
21761 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
21764 AllocatedPhysRegs.push_back(RISCV::V0);
21770 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
21771 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
21772 StartReg += RegsNeeded) {
21773 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
21774 if ((AssignedMap & Map) == 0) {
21775 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
21776 AssignedMap |=
Map;
21781 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
21784 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
21785 allocate(RVVArgInfos[i]);
21789 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
21790 return AllocatedPhysRegs[CurIdx++];
21797 if (Subtarget.hasStdExtZicfilp()) {
21809#define GET_RISCVVIntrinsicsTable_IMPL
21810#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static bool hasPassthruOp(unsigned Opcode)
Return true if a RISC-V target specified op has a passthru operand.
static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< APInt > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static unsigned getPACKOpcode(unsigned DestBW, const RISCVSubtarget &Subtarget)
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineOp_VLToVWOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary or FMA operation to its equivalent VW or VW_W form.
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static SDValue lowerBUILD_VECTORvXf16(SDValue Op, SelectionDAG &DAG)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue performVFMADD_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Double the element size of the build vector to reduce the number of vslide1down in the build vector c...
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
void clearAllBits()
Set every bit to 0.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
bool hasVInstructionsBF16Minimal() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool isTargetAndroid() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI, SmallVectorImpl< Instruction * > &DeadInsts) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI, SmallVectorImpl< Instruction * > &DeadInsts) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SW_GUARDED_BRIND
Software guarded BRIND node.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ TRUNCATE_VECTOR_VL_USAT
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ TRUNCATE_VECTOR_VL_SSAT
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost, bool FreeZeroes)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)