38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
87 !Subtarget.hasStdExtF()) {
88 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
89 "doesn't support the F instruction set extension (ignoring "
93 !Subtarget.hasStdExtD()) {
94 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
95 "doesn't support the D instruction set extension (ignoring "
119 if (Subtarget.hasStdExtZfhmin())
121 if (Subtarget.hasStdExtZfbfmin())
123 if (Subtarget.hasStdExtF())
125 if (Subtarget.hasStdExtD())
127 if (Subtarget.hasStdExtZhinxmin())
129 if (Subtarget.hasStdExtZfinx())
131 if (Subtarget.hasStdExtZdinx()) {
139 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
140 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
142 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
143 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
144 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
145 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
146 MVT::nxv4i64, MVT::nxv8i64};
148 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
149 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
151 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
152 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
154 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
156 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
159 auto addRegClassForRVV = [
this](
MVT VT) {
163 if (VT.getVectorMinNumElements() < MinElts)
166 unsigned Size = VT.getSizeInBits().getKnownMinValue();
169 RC = &RISCV::VRRegClass;
171 RC = &RISCV::VRM2RegClass;
173 RC = &RISCV::VRM4RegClass;
175 RC = &RISCV::VRM8RegClass;
182 for (
MVT VT : BoolVecVTs)
183 addRegClassForRVV(VT);
184 for (
MVT VT : IntVecVTs) {
185 if (VT.getVectorElementType() == MVT::i64 &&
188 addRegClassForRVV(VT);
192 for (
MVT VT : F16VecVTs)
193 addRegClassForRVV(VT);
196 for (
MVT VT : BF16VecVTs)
197 addRegClassForRVV(VT);
200 for (
MVT VT : F32VecVTs)
201 addRegClassForRVV(VT);
204 for (
MVT VT : F64VecVTs)
205 addRegClassForRVV(VT);
208 auto addRegClassForFixedVectors = [
this](
MVT VT) {
215 if (useRVVForFixedLengthVectorVT(VT))
216 addRegClassForFixedVectors(VT);
219 if (useRVVForFixedLengthVectorVT(VT))
220 addRegClassForFixedVectors(VT);
247 if (!(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
261 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
262 !(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()))
272 if (!Subtarget.hasStdExtZbb())
278 if (!Subtarget.hasStdExtZmmul()) {
280 }
else if (Subtarget.
is64Bit()) {
287 if (!Subtarget.hasStdExtM()) {
290 }
else if (Subtarget.
is64Bit()) {
292 {MVT::i8, MVT::i16, MVT::i32},
Custom);
302 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
305 }
else if (Subtarget.hasVendorXTHeadBb()) {
309 }
else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
318 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
319 Subtarget.hasVendorXTHeadBb())
323 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
331 if (Subtarget.hasStdExtZbb() ||
332 (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
337 if (Subtarget.hasStdExtZbb() ||
338 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
345 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
346 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
355 if (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()) {
357 }
else if (Subtarget.hasShortForwardBranchOpt()) {
360 }
else if (Subtarget.
is64Bit()) {
364 if (!Subtarget.hasVendorXTHeadCondMov())
367 static const unsigned FPLegalNodeTypes[] = {
380 static const unsigned FPOpToExpand[] = {
384 static const unsigned FPRndMode[] = {
391 static const unsigned ZfhminZfbfminPromoteOps[] = {
402 if (Subtarget.hasStdExtZfbfmin()) {
485 if (Subtarget.hasStdExtZfa()) {
502 if (Subtarget.hasStdExtZfa()) {
574 if (Subtarget.hasStdExtZicbop()) {
578 if (Subtarget.hasStdExtA()) {
580 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
584 }
else if (Subtarget.hasForcedAtomics()) {
608 {MVT::i8, MVT::i16},
Custom);
619 static const unsigned IntegerVPOps[] = {
620 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
621 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
622 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
623 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
624 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
625 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
626 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
627 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
628 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
629 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
630 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
631 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
632 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
633 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
634 ISD::EXPERIMENTAL_VP_SPLAT};
636 static const unsigned FloatingPointVPOps[] = {
637 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
638 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
639 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
640 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
641 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
642 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
643 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
644 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
645 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
646 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
647 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
648 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
649 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
650 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
652 static const unsigned IntegerVecReduceOps[] = {
657 static const unsigned FloatingPointVecReduceOps[] = {
670 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
671 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
672 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
676 for (
MVT VT : BoolVecVTs) {
706 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
730 ISD::VP_TRUNCATE, ISD::VP_SETCC},
746 for (
MVT VT : IntVecVTs) {
757 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
807 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
808 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
832 if (Subtarget.hasStdExtZvkb()) {
840 if (Subtarget.hasStdExtZvbb()) {
844 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
850 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
859 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
879 static const unsigned ZvfhminPromoteOps[] = {
889 static const unsigned ZvfhminPromoteVPOps[] = {
890 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
891 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
892 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
893 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
894 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
895 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
896 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
897 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
898 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
901 const auto SetCommonVFPActions = [&](
MVT VT) {
946 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
947 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
978 const auto SetCommonVFPExtLoadTruncStoreActions =
980 for (
auto SmallVT : SmallerVTs) {
987 for (
MVT VT : F16VecVTs) {
990 SetCommonVFPActions(VT);
993 for (
MVT VT : F16VecVTs) {
1004 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1009 if (Subtarget.hasStdExtZfhmin())
1015 if (VT == MVT::nxv32f16) {
1029 for (
MVT VT : BF16VecVTs) {
1040 if (Subtarget.hasStdExtZfbfmin())
1050 for (
MVT VT : F32VecVTs) {
1053 SetCommonVFPActions(VT);
1054 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1059 for (
MVT VT : F64VecVTs) {
1062 SetCommonVFPActions(VT);
1063 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1064 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1070 if (!useRVVForFixedLengthVectorVT(VT))
1115 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1142 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1164 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1165 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1202 if (Subtarget.hasStdExtZvkb())
1205 if (Subtarget.hasStdExtZvbb()) {
1227 if (!useRVVForFixedLengthVectorVT(VT))
1257 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1260 if (Subtarget.hasStdExtZfhmin()) {
1298 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1299 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1345 if (Subtarget.hasStdExtA())
1348 if (Subtarget.hasForcedAtomics()) {
1358 if (Subtarget.hasVendorXTHeadMemIdx()) {
1374 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
1399 if (Subtarget.hasStdExtZbb())
1402 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1406 if (Subtarget.hasStdExtZbkb())
1418 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1421 if (Subtarget.hasVendorXTHeadMemPair())
1450MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1455bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1457 bool IsScalable)
const {
1464 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1489 unsigned Intrinsic)
const {
1490 auto &
DL =
I.getDataLayout();
1492 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1493 bool IsUnitStrided,
bool UsePtrVal =
false) {
1498 Info.ptrVal =
I.getArgOperand(PtrOp);
1500 Info.fallbackAddressSpace =
1501 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1505 MemTy =
I.getArgOperand(0)->getType();
1508 MemTy =
I.getType();
1523 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1527 switch (Intrinsic) {
1530 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1531 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1532 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1533 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1534 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1535 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1536 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1537 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1538 case Intrinsic::riscv_masked_cmpxchg_i32:
1540 Info.memVT = MVT::i32;
1541 Info.ptrVal =
I.getArgOperand(0);
1547 case Intrinsic::riscv_seg2_load:
1548 case Intrinsic::riscv_seg3_load:
1549 case Intrinsic::riscv_seg4_load:
1550 case Intrinsic::riscv_seg5_load:
1551 case Intrinsic::riscv_seg6_load:
1552 case Intrinsic::riscv_seg7_load:
1553 case Intrinsic::riscv_seg8_load:
1554 return SetRVVLoadStoreInfo( 0,
false,
1556 case Intrinsic::riscv_seg2_store:
1557 case Intrinsic::riscv_seg3_store:
1558 case Intrinsic::riscv_seg4_store:
1559 case Intrinsic::riscv_seg5_store:
1560 case Intrinsic::riscv_seg6_store:
1561 case Intrinsic::riscv_seg7_store:
1562 case Intrinsic::riscv_seg8_store:
1564 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1567 case Intrinsic::riscv_vle:
1568 case Intrinsic::riscv_vle_mask:
1569 case Intrinsic::riscv_vleff:
1570 case Intrinsic::riscv_vleff_mask:
1571 return SetRVVLoadStoreInfo( 1,
1575 case Intrinsic::riscv_vse:
1576 case Intrinsic::riscv_vse_mask:
1577 return SetRVVLoadStoreInfo( 1,
1581 case Intrinsic::riscv_vlse:
1582 case Intrinsic::riscv_vlse_mask:
1583 case Intrinsic::riscv_vloxei:
1584 case Intrinsic::riscv_vloxei_mask:
1585 case Intrinsic::riscv_vluxei:
1586 case Intrinsic::riscv_vluxei_mask:
1587 return SetRVVLoadStoreInfo( 1,
1590 case Intrinsic::riscv_vsse:
1591 case Intrinsic::riscv_vsse_mask:
1592 case Intrinsic::riscv_vsoxei:
1593 case Intrinsic::riscv_vsoxei_mask:
1594 case Intrinsic::riscv_vsuxei:
1595 case Intrinsic::riscv_vsuxei_mask:
1596 return SetRVVLoadStoreInfo( 1,
1599 case Intrinsic::riscv_vlseg2:
1600 case Intrinsic::riscv_vlseg3:
1601 case Intrinsic::riscv_vlseg4:
1602 case Intrinsic::riscv_vlseg5:
1603 case Intrinsic::riscv_vlseg6:
1604 case Intrinsic::riscv_vlseg7:
1605 case Intrinsic::riscv_vlseg8:
1606 case Intrinsic::riscv_vlseg2ff:
1607 case Intrinsic::riscv_vlseg3ff:
1608 case Intrinsic::riscv_vlseg4ff:
1609 case Intrinsic::riscv_vlseg5ff:
1610 case Intrinsic::riscv_vlseg6ff:
1611 case Intrinsic::riscv_vlseg7ff:
1612 case Intrinsic::riscv_vlseg8ff:
1613 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1616 case Intrinsic::riscv_vlseg2_mask:
1617 case Intrinsic::riscv_vlseg3_mask:
1618 case Intrinsic::riscv_vlseg4_mask:
1619 case Intrinsic::riscv_vlseg5_mask:
1620 case Intrinsic::riscv_vlseg6_mask:
1621 case Intrinsic::riscv_vlseg7_mask:
1622 case Intrinsic::riscv_vlseg8_mask:
1623 case Intrinsic::riscv_vlseg2ff_mask:
1624 case Intrinsic::riscv_vlseg3ff_mask:
1625 case Intrinsic::riscv_vlseg4ff_mask:
1626 case Intrinsic::riscv_vlseg5ff_mask:
1627 case Intrinsic::riscv_vlseg6ff_mask:
1628 case Intrinsic::riscv_vlseg7ff_mask:
1629 case Intrinsic::riscv_vlseg8ff_mask:
1630 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1633 case Intrinsic::riscv_vlsseg2:
1634 case Intrinsic::riscv_vlsseg3:
1635 case Intrinsic::riscv_vlsseg4:
1636 case Intrinsic::riscv_vlsseg5:
1637 case Intrinsic::riscv_vlsseg6:
1638 case Intrinsic::riscv_vlsseg7:
1639 case Intrinsic::riscv_vlsseg8:
1640 case Intrinsic::riscv_vloxseg2:
1641 case Intrinsic::riscv_vloxseg3:
1642 case Intrinsic::riscv_vloxseg4:
1643 case Intrinsic::riscv_vloxseg5:
1644 case Intrinsic::riscv_vloxseg6:
1645 case Intrinsic::riscv_vloxseg7:
1646 case Intrinsic::riscv_vloxseg8:
1647 case Intrinsic::riscv_vluxseg2:
1648 case Intrinsic::riscv_vluxseg3:
1649 case Intrinsic::riscv_vluxseg4:
1650 case Intrinsic::riscv_vluxseg5:
1651 case Intrinsic::riscv_vluxseg6:
1652 case Intrinsic::riscv_vluxseg7:
1653 case Intrinsic::riscv_vluxseg8:
1654 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1657 case Intrinsic::riscv_vlsseg2_mask:
1658 case Intrinsic::riscv_vlsseg3_mask:
1659 case Intrinsic::riscv_vlsseg4_mask:
1660 case Intrinsic::riscv_vlsseg5_mask:
1661 case Intrinsic::riscv_vlsseg6_mask:
1662 case Intrinsic::riscv_vlsseg7_mask:
1663 case Intrinsic::riscv_vlsseg8_mask:
1664 case Intrinsic::riscv_vloxseg2_mask:
1665 case Intrinsic::riscv_vloxseg3_mask:
1666 case Intrinsic::riscv_vloxseg4_mask:
1667 case Intrinsic::riscv_vloxseg5_mask:
1668 case Intrinsic::riscv_vloxseg6_mask:
1669 case Intrinsic::riscv_vloxseg7_mask:
1670 case Intrinsic::riscv_vloxseg8_mask:
1671 case Intrinsic::riscv_vluxseg2_mask:
1672 case Intrinsic::riscv_vluxseg3_mask:
1673 case Intrinsic::riscv_vluxseg4_mask:
1674 case Intrinsic::riscv_vluxseg5_mask:
1675 case Intrinsic::riscv_vluxseg6_mask:
1676 case Intrinsic::riscv_vluxseg7_mask:
1677 case Intrinsic::riscv_vluxseg8_mask:
1678 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1681 case Intrinsic::riscv_vsseg2:
1682 case Intrinsic::riscv_vsseg3:
1683 case Intrinsic::riscv_vsseg4:
1684 case Intrinsic::riscv_vsseg5:
1685 case Intrinsic::riscv_vsseg6:
1686 case Intrinsic::riscv_vsseg7:
1687 case Intrinsic::riscv_vsseg8:
1688 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1691 case Intrinsic::riscv_vsseg2_mask:
1692 case Intrinsic::riscv_vsseg3_mask:
1693 case Intrinsic::riscv_vsseg4_mask:
1694 case Intrinsic::riscv_vsseg5_mask:
1695 case Intrinsic::riscv_vsseg6_mask:
1696 case Intrinsic::riscv_vsseg7_mask:
1697 case Intrinsic::riscv_vsseg8_mask:
1698 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1701 case Intrinsic::riscv_vssseg2:
1702 case Intrinsic::riscv_vssseg3:
1703 case Intrinsic::riscv_vssseg4:
1704 case Intrinsic::riscv_vssseg5:
1705 case Intrinsic::riscv_vssseg6:
1706 case Intrinsic::riscv_vssseg7:
1707 case Intrinsic::riscv_vssseg8:
1708 case Intrinsic::riscv_vsoxseg2:
1709 case Intrinsic::riscv_vsoxseg3:
1710 case Intrinsic::riscv_vsoxseg4:
1711 case Intrinsic::riscv_vsoxseg5:
1712 case Intrinsic::riscv_vsoxseg6:
1713 case Intrinsic::riscv_vsoxseg7:
1714 case Intrinsic::riscv_vsoxseg8:
1715 case Intrinsic::riscv_vsuxseg2:
1716 case Intrinsic::riscv_vsuxseg3:
1717 case Intrinsic::riscv_vsuxseg4:
1718 case Intrinsic::riscv_vsuxseg5:
1719 case Intrinsic::riscv_vsuxseg6:
1720 case Intrinsic::riscv_vsuxseg7:
1721 case Intrinsic::riscv_vsuxseg8:
1722 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1725 case Intrinsic::riscv_vssseg2_mask:
1726 case Intrinsic::riscv_vssseg3_mask:
1727 case Intrinsic::riscv_vssseg4_mask:
1728 case Intrinsic::riscv_vssseg5_mask:
1729 case Intrinsic::riscv_vssseg6_mask:
1730 case Intrinsic::riscv_vssseg7_mask:
1731 case Intrinsic::riscv_vssseg8_mask:
1732 case Intrinsic::riscv_vsoxseg2_mask:
1733 case Intrinsic::riscv_vsoxseg3_mask:
1734 case Intrinsic::riscv_vsoxseg4_mask:
1735 case Intrinsic::riscv_vsoxseg5_mask:
1736 case Intrinsic::riscv_vsoxseg6_mask:
1737 case Intrinsic::riscv_vsoxseg7_mask:
1738 case Intrinsic::riscv_vsoxseg8_mask:
1739 case Intrinsic::riscv_vsuxseg2_mask:
1740 case Intrinsic::riscv_vsuxseg3_mask:
1741 case Intrinsic::riscv_vsuxseg4_mask:
1742 case Intrinsic::riscv_vsuxseg5_mask:
1743 case Intrinsic::riscv_vsuxseg6_mask:
1744 case Intrinsic::riscv_vsuxseg7_mask:
1745 case Intrinsic::riscv_vsuxseg8_mask:
1746 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1787 return isInt<12>(Imm);
1791 return isInt<12>(Imm);
1804 return (SrcBits == 64 && DestBits == 32);
1815 return (SrcBits == 64 && DestBits == 32);
1826 if (SrcBits == DestBits * 2) {
1837 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1838 EVT MemVT = LD->getMemoryVT();
1839 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1849 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1857 return Subtarget.hasStdExtZbb() ||
1858 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1862 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1863 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1874 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1879 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1883 EVT VT =
Y.getValueType();
1889 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1890 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
1895 if (Subtarget.hasStdExtZbs())
1896 return X.getValueType().isScalarInteger();
1897 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1899 if (Subtarget.hasVendorXTHeadBs())
1900 return C !=
nullptr;
1902 return C &&
C->getAPIntValue().ule(10);
1922 if (BitSize > Subtarget.
getXLen())
1926 int64_t Val = Imm.getSExtValue();
1934 if (!Subtarget.enableUnalignedScalarMem())
1950 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1957 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1961 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1973 case Instruction::Add:
1974 case Instruction::Sub:
1975 case Instruction::Mul:
1976 case Instruction::And:
1977 case Instruction::Or:
1978 case Instruction::Xor:
1979 case Instruction::FAdd:
1980 case Instruction::FSub:
1981 case Instruction::FMul:
1982 case Instruction::FDiv:
1983 case Instruction::ICmp:
1984 case Instruction::FCmp:
1986 case Instruction::Shl:
1987 case Instruction::LShr:
1988 case Instruction::AShr:
1989 case Instruction::UDiv:
1990 case Instruction::SDiv:
1991 case Instruction::URem:
1992 case Instruction::SRem:
1993 case Instruction::Select:
1994 return Operand == 1;
2008 auto *
II = dyn_cast<IntrinsicInst>(
I);
2012 switch (
II->getIntrinsicID()) {
2013 case Intrinsic::fma:
2014 case Intrinsic::vp_fma:
2015 return Operand == 0 || Operand == 1;
2016 case Intrinsic::vp_shl:
2017 case Intrinsic::vp_lshr:
2018 case Intrinsic::vp_ashr:
2019 case Intrinsic::vp_udiv:
2020 case Intrinsic::vp_sdiv:
2021 case Intrinsic::vp_urem:
2022 case Intrinsic::vp_srem:
2023 case Intrinsic::ssub_sat:
2024 case Intrinsic::vp_ssub_sat:
2025 case Intrinsic::usub_sat:
2026 case Intrinsic::vp_usub_sat:
2027 return Operand == 1;
2029 case Intrinsic::vp_add:
2030 case Intrinsic::vp_mul:
2031 case Intrinsic::vp_and:
2032 case Intrinsic::vp_or:
2033 case Intrinsic::vp_xor:
2034 case Intrinsic::vp_fadd:
2035 case Intrinsic::vp_fmul:
2036 case Intrinsic::vp_icmp:
2037 case Intrinsic::vp_fcmp:
2038 case Intrinsic::smin:
2039 case Intrinsic::vp_smin:
2040 case Intrinsic::umin:
2041 case Intrinsic::vp_umin:
2042 case Intrinsic::smax:
2043 case Intrinsic::vp_smax:
2044 case Intrinsic::umax:
2045 case Intrinsic::vp_umax:
2046 case Intrinsic::sadd_sat:
2047 case Intrinsic::vp_sadd_sat:
2048 case Intrinsic::uadd_sat:
2049 case Intrinsic::vp_uadd_sat:
2051 case Intrinsic::vp_sub:
2052 case Intrinsic::vp_fsub:
2053 case Intrinsic::vp_fdiv:
2054 return Operand == 0 || Operand == 1;
2075 if (!Subtarget.sinkSplatOperands())
2078 for (
auto OpIdx :
enumerate(
I->operands())) {
2082 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2084 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2093 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2098 for (
Use &U :
Op->uses()) {
2150 if (!Subtarget.hasStdExtZfa())
2151 return std::make_pair(-1,
false);
2153 bool IsSupportedVT =
false;
2154 if (VT == MVT::f16) {
2155 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2156 }
else if (VT == MVT::f32) {
2157 IsSupportedVT =
true;
2158 }
else if (VT == MVT::f64) {
2159 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2160 IsSupportedVT =
true;
2164 return std::make_pair(-1,
false);
2167 if (
Index < 0 && Imm.isNegative())
2171 return std::make_pair(
Index,
false);
2175 bool ForCodeSize)
const {
2176 bool IsLegalVT =
false;
2179 else if (VT == MVT::f32)
2181 else if (VT == MVT::f64)
2183 else if (VT == MVT::bf16)
2184 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2196 return Imm.isZero();
2200 if (Imm.isNegZero())
2213 unsigned Index)
const {
2226 if (EltVT == MVT::i1)
2239 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2246 if ((ResElts * 2) != SrcElts)
2287 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2289 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2304 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2310 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2323 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2324 int64_t
C = RHSC->getSExtValue();
2366 switch (KnownSize) {
2394 return RISCV::VRRegClassID;
2396 return RISCV::VRM2RegClassID;
2398 return RISCV::VRM4RegClassID;
2400 return RISCV::VRM8RegClassID;
2410 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2411 "Unexpected subreg numbering");
2412 return RISCV::sub_vrm1_0 +
Index;
2415 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2416 "Unexpected subreg numbering");
2417 return RISCV::sub_vrm2_0 +
Index;
2420 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2421 "Unexpected subreg numbering");
2422 return RISCV::sub_vrm4_0 +
Index;
2429 return RISCV::VRRegClassID;
2438std::pair<unsigned, unsigned>
2440 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2442 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2443 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2444 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2445 "Register classes not ordered");
2454 unsigned SubRegIdx = RISCV::NoSubRegister;
2455 for (
const unsigned RCID :
2456 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2457 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2461 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2466 return {SubRegIdx, InsertExtractIdx};
2471bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2500unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2507 "Unexpected opcode");
2509 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2511 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2514 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2588bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2589 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2598 "Expected legal fixed length vector!");
2601 unsigned MaxELen = Subtarget.
getELen();
2635 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2642 "Expected to convert into a scalable vector!");
2643 assert(V.getValueType().isFixedLengthVector() &&
2644 "Expected a fixed length vector operand!");
2654 "Expected to convert into a fixed length vector!");
2655 assert(V.getValueType().isScalableVector() &&
2656 "Expected a scalable vector operand!");
2679static std::pair<SDValue, SDValue>
2688static std::pair<SDValue, SDValue>
2701static std::pair<SDValue, SDValue>
2718std::pair<unsigned, unsigned>
2734 return std::make_pair(MinVLMAX, MaxVLMAX);
2746 EVT VT,
unsigned DefinedValues)
const {
2760 std::tie(LMul, Fractional) =
2763 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2765 Cost = (LMul * DLenFactor);
2810 MVT DstVT =
Op.getSimpleValueType();
2811 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2819 Src.getValueType() == MVT::bf16) {
2826 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2834 Opc,
DL, DstVT, Src,
2848 MVT SrcVT = Src.getSimpleValueType();
2854 if (SatVT != DstEltVT)
2857 MVT DstContainerVT = DstVT;
2858 MVT SrcContainerVT = SrcVT;
2864 "Expected same element count");
2873 {Src, Src, DAG.getCondCode(ISD::SETNE),
2874 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2878 if (DstEltSize > (2 * SrcEltSize)) {
2884 MVT CvtContainerVT = DstContainerVT;
2885 MVT CvtEltVT = DstEltVT;
2886 if (SrcEltSize > (2 * DstEltSize)) {
2895 while (CvtContainerVT != DstContainerVT) {
2901 Res = DAG.
getNode(ClipOpc,
DL, CvtContainerVT, Res, Mask, VL);
2908 Res, DAG.
getUNDEF(DstContainerVT), VL);
2920 case ISD::VP_FROUNDEVEN:
2924 case ISD::VP_FROUNDTOZERO:
2928 case ISD::VP_FFLOOR:
2936 case ISD::VP_FROUND:
2952 MVT VT =
Op.getSimpleValueType();
2959 MVT ContainerVT = VT;
2966 if (
Op->isVPOpcode()) {
2967 Mask =
Op.getOperand(1);
2971 VL =
Op.getOperand(2);
2993 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3007 switch (
Op.getOpcode()) {
3013 case ISD::VP_FFLOOR:
3016 case ISD::VP_FROUND:
3017 case ISD::VP_FROUNDEVEN:
3018 case ISD::VP_FROUNDTOZERO: {
3034 case ISD::VP_FNEARBYINT:
3047 Src, Src, Mask, VL);
3062 MVT VT =
Op.getSimpleValueType();
3066 MVT ContainerVT = VT;
3078 MVT MaskVT = Mask.getSimpleValueType();
3081 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3082 DAG.getUNDEF(MaskVT), Mask, VL});
3086 {Chain, Src, Src, Src, Unorder, VL});
3087 Chain = Src.getValue(1);
3103 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3115 switch (
Op.getOpcode()) {
3126 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3132 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3136 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3145 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3146 Truncated, Mask, VL);
3152 Src, Src, Mask, VL);
3162 MVT VT =
Op.getSimpleValueType();
3190 MVT VT =
Op.getSimpleValueType();
3195 MVT ContainerVT = VT;
3238 "Unexpected vector MVT");
3266 return std::nullopt;
3284 unsigned EltSizeInBits) {
3287 return std::nullopt;
3288 bool IsInteger =
Op.getValueType().isInteger();
3290 std::optional<unsigned> SeqStepDenom;
3291 std::optional<APInt> SeqStepNum;
3292 std::optional<APInt> SeqAddend;
3293 std::optional<std::pair<APInt, unsigned>> PrevElt;
3294 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3299 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3301 if (Elt.isUndef()) {
3302 Elts[
Idx] = std::nullopt;
3306 Elts[
Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
3311 return std::nullopt;
3312 Elts[
Idx] = *ExactInteger;
3325 unsigned IdxDiff =
Idx - PrevElt->second;
3326 APInt ValDiff = *Elt - PrevElt->first;
3334 int64_t Remainder = ValDiff.
srem(IdxDiff);
3339 return std::nullopt;
3340 ValDiff = ValDiff.
sdiv(IdxDiff);
3345 SeqStepNum = ValDiff;
3346 else if (ValDiff != SeqStepNum)
3347 return std::nullopt;
3350 SeqStepDenom = IdxDiff;
3351 else if (IdxDiff != *SeqStepDenom)
3352 return std::nullopt;
3356 if (!PrevElt || PrevElt->first != *Elt)
3357 PrevElt = std::make_pair(*Elt,
Idx);
3361 if (!SeqStepNum || !SeqStepDenom)
3362 return std::nullopt;
3370 (
APInt(EltSizeInBits,
Idx) * *SeqStepNum).sdiv(*SeqStepDenom);
3372 APInt Addend = *Elt - ExpectedVal;
3375 else if (Addend != SeqAddend)
3376 return std::nullopt;
3379 assert(SeqAddend &&
"Must have an addend if we have a step");
3381 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
3382 SeqAddend->getSExtValue()};
3403 MVT ContainerVT = VT;
3431 MVT VT =
Op.getSimpleValueType();
3443 unsigned MostCommonCount = 0;
3445 unsigned NumUndefElts =
3453 unsigned NumScalarLoads = 0;
3459 ValueCounts.
insert(std::make_pair(V, 0));
3460 unsigned &Count = ValueCounts[V];
3462 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3463 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3468 if (++Count >= MostCommonCount) {
3470 MostCommonCount = Count;
3474 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3475 unsigned NumDefElts = NumElts - NumUndefElts;
3476 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3482 ((MostCommonCount > DominantValueCountThreshold) ||
3495 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3496 LastOp != DominantValue) {
3505 Processed.insert(LastOp);
3510 const SDValue &V = OpIdx.value();
3511 if (V.isUndef() || !Processed.insert(V).second)
3513 if (ValueCounts[V] == 1) {
3522 return DAG.getConstant(V == V1, DL, XLenVT);
3538 MVT VT =
Op.getSimpleValueType();
3568 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3569 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3577 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3578 MVT IntegerViaVecVT =
3583 unsigned BitPos = 0, IntegerEltIdx = 0;
3586 for (
unsigned I = 0;
I < NumElts;) {
3588 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3589 Bits |= ((
uint64_t)BitValue << BitPos);
3595 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3596 if (NumViaIntegerBits <= 32)
3597 Bits = SignExtend64<32>(Bits);
3599 Elts[IntegerEltIdx] = Elt;
3608 if (NumElts < NumViaIntegerBits) {
3612 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3640 int64_t StepNumerator = SimpleVID->StepNumerator;
3641 unsigned StepDenominator = SimpleVID->StepDenominator;
3642 int64_t Addend = SimpleVID->Addend;
3644 assert(StepNumerator != 0 &&
"Invalid step");
3645 bool Negate =
false;
3646 int64_t SplatStepVal = StepNumerator;
3650 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3652 Negate = StepNumerator < 0;
3654 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3661 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3662 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3664 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3667 MVT VIDContainerVT =
3675 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3676 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3678 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3680 if (StepDenominator != 1) {
3685 if (Addend != 0 || Negate) {
3704 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3705 "Unexpected sequence type");
3709 unsigned ViaVecLen =
3713 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3716 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3717 const auto &SeqV = OpIdx.value();
3718 if (!SeqV.isUndef())
3720 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3725 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3726 SplatValue = SignExtend64<32>(SplatValue);
3748 const auto *BV = cast<BuildVectorSDNode>(
Op);
3751 BV->getRepeatedSequence(Sequence) &&
3752 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3753 unsigned SeqLen = Sequence.size();
3755 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3756 ViaIntVT == MVT::i64) &&
3757 "Unexpected sequence type");
3762 const unsigned RequiredVL = NumElts / SeqLen;
3763 const unsigned ViaVecLen =
3765 NumElts : RequiredVL;
3768 unsigned EltIdx = 0;
3769 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3773 for (
const auto &SeqV : Sequence) {
3774 if (!SeqV.isUndef())
3776 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3782 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3783 SplatValue = SignExtend64<32>(SplatValue);
3789 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3790 "Unexpected bitcast sequence");
3791 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3794 MVT ViaContainerVT =
3801 if (ViaVecLen != RequiredVL)
3821 Source, DAG, Subtarget);
3841 return RISCV::PACKH;
3843 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
3858 MVT VT =
Op.getSimpleValueType();
3866 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
3871 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
3885 if (Subtarget.hasStdExtZbkb())
3890 ElemDL, XLenVT,
A,
B),
3897 Flags.setDisjoint(
true);
3903 NewOperands.
reserve(NumElts / 2);
3905 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
3915 MVT VT =
Op.getSimpleValueType();
3926 MVT VT =
Op.getSimpleValueType();
3931 !Subtarget.hasStdExtZfhmin())
4003 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
4007 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4030 unsigned NumUndefElts =
4032 unsigned NumDefElts = NumElts - NumUndefElts;
4033 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4040 for (
unsigned i = 0; i < NumElts; i++) {
4042 if (i < NumElts / 2) {
4049 bool SelectMaskVal = (i < NumElts / 2);
4052 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4053 MaskVals.
size() == NumElts);
4088 unsigned UndefCount = 0;
4095 LinearBudget -= PerSlideCost;
4098 LinearBudget -= PerSlideCost;
4101 LinearBudget -= PerSlideCost;
4104 if (LinearBudget < 0)
4109 "Illegal type which will result in reserved encoding");
4134 Vec,
Offset, Mask, VL, Policy);
4147 Vec,
Offset, Mask, VL, Policy);
4157 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4158 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4159 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4162 if ((LoC >> 31) == HiC)
4173 (isa<RegisterSDNode>(VL) &&
4174 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4176 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4191 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4192 Hi.getConstantOperandVal(1) == 31)