38#include "llvm/IR/IntrinsicsRISCV.h"
53#define DEBUG_TYPE "riscv-lower"
59 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
60 "instructions that we will consider for VW expansion"),
65 cl::desc(
"Allow the formation of VW_W operations (e.g., "
66 "VWADD_W) with splat constants"),
71 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
72 "transformation to multiplications by the reciprocal"),
77 cl::desc(
"Give the maximum number of instructions that we will "
78 "use for creating a floating-point immediate value"),
89 !Subtarget.hasStdExtF()) {
90 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
91 "doesn't support the F instruction set extension (ignoring "
95 !Subtarget.hasStdExtD()) {
96 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
97 "doesn't support the D instruction set extension (ignoring "
121 if (Subtarget.hasStdExtZfhmin())
123 if (Subtarget.hasStdExtZfbfmin())
125 if (Subtarget.hasStdExtF())
127 if (Subtarget.hasStdExtD())
129 if (Subtarget.hasStdExtZhinxmin())
131 if (Subtarget.hasStdExtZfinx())
133 if (Subtarget.hasStdExtZdinx()) {
141 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
142 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
144 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
145 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
146 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
147 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
148 MVT::nxv4i64, MVT::nxv8i64};
150 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
151 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
153 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
154 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
156 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
158 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
160 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
161 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
162 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
163 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
164 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
165 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
166 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
167 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
168 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
169 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
170 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
173 auto addRegClassForRVV = [
this](
MVT VT) {
177 if (VT.getVectorMinNumElements() < MinElts)
180 unsigned Size = VT.getSizeInBits().getKnownMinValue();
183 RC = &RISCV::VRRegClass;
185 RC = &RISCV::VRM2RegClass;
187 RC = &RISCV::VRM4RegClass;
189 RC = &RISCV::VRM8RegClass;
196 for (
MVT VT : BoolVecVTs)
197 addRegClassForRVV(VT);
198 for (
MVT VT : IntVecVTs) {
199 if (VT.getVectorElementType() == MVT::i64 &&
202 addRegClassForRVV(VT);
206 for (
MVT VT : F16VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : BF16VecVTs)
211 addRegClassForRVV(VT);
214 for (
MVT VT : F32VecVTs)
215 addRegClassForRVV(VT);
218 for (
MVT VT : F64VecVTs)
219 addRegClassForRVV(VT);
222 auto addRegClassForFixedVectors = [
this](
MVT VT) {
229 if (useRVVForFixedLengthVectorVT(VT))
230 addRegClassForFixedVectors(VT);
233 if (useRVVForFixedLengthVectorVT(VT))
234 addRegClassForFixedVectors(VT);
294 if (!(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
304 if (!Subtarget.hasVendorXTHeadBb())
309 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
310 !(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()))
320 if (!Subtarget.hasStdExtZbb())
326 if (!Subtarget.hasStdExtZmmul()) {
328 }
else if (Subtarget.
is64Bit()) {
335 if (!Subtarget.hasStdExtM()) {
338 }
else if (Subtarget.
is64Bit()) {
340 {MVT::i8, MVT::i16, MVT::i32},
Custom);
350 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
353 }
else if (Subtarget.hasVendorXTHeadBb()) {
357 }
else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
366 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
367 Subtarget.hasVendorXTHeadBb())
371 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
379 if (Subtarget.hasStdExtZbb() ||
380 (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
385 if (Subtarget.hasStdExtZbb() ||
386 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
393 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
394 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
403 if (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()) {
405 }
else if (Subtarget.hasShortForwardBranchOpt()) {
408 }
else if (Subtarget.
is64Bit()) {
412 if (!Subtarget.hasVendorXTHeadCondMov())
415 static const unsigned FPLegalNodeTypes[] = {
429 static const unsigned FPOpToExpand[] = {
433 static const unsigned FPRndMode[] = {
437 static const unsigned ZfhminZfbfminPromoteOps[] = {
448 if (Subtarget.hasStdExtZfbfmin()) {
471 if (Subtarget.hasStdExtZfa())
498 Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ?
Legal :
Promote);
540 if (Subtarget.hasStdExtZfa()) {
558 if (Subtarget.hasStdExtZfa()) {
640 if (Subtarget.hasStdExtZicbop()) {
644 if (Subtarget.hasStdExtA()) {
646 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
650 }
else if (Subtarget.hasForcedAtomics()) {
674 {MVT::i8, MVT::i16},
Custom);
685 static const unsigned IntegerVPOps[] = {
686 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
687 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
688 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
689 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
690 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
691 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
692 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
693 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
694 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
695 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
696 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
697 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
698 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
699 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
700 ISD::EXPERIMENTAL_VP_SPLAT};
702 static const unsigned FloatingPointVPOps[] = {
703 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
704 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
705 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
706 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
707 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
708 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
709 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
710 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
711 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
712 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
713 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
714 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
715 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
716 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
718 static const unsigned IntegerVecReduceOps[] = {
723 static const unsigned FloatingPointVecReduceOps[] = {
727 static const unsigned FloatingPointLibCallOps[] = {
740 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
741 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
742 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
746 for (
MVT VT : BoolVecVTs) {
776 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
800 ISD::VP_TRUNCATE, ISD::VP_SETCC},
816 for (
MVT VT : IntVecVTs) {
827 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
877 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
878 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
902 if (Subtarget.hasStdExtZvkb()) {
910 if (Subtarget.hasStdExtZvbb()) {
914 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
920 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
929 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
937 for (
MVT VT : VecTupleVTs) {
958 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
968 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
983 ISD::VP_FROUNDTOZERO,
989 ISD::VP_REDUCE_FMINIMUM,
990 ISD::VP_REDUCE_FMAXIMUM};
993 const auto SetCommonVFPActions = [&](
MVT VT) {
1028 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1029 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1062 const auto SetCommonVFPExtLoadTruncStoreActions =
1064 for (
auto SmallVT : SmallerVTs) {
1072 const auto SetCommonPromoteToF32Actions = [&](
MVT VT) {
1097 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1098 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1121 for (
MVT VT : F16VecVTs) {
1124 SetCommonVFPActions(VT);
1127 for (
MVT VT : F16VecVTs) {
1130 SetCommonPromoteToF32Actions(VT);
1135 for (
MVT VT : BF16VecVTs) {
1138 SetCommonPromoteToF32Actions(VT);
1143 for (
MVT VT : F32VecVTs) {
1146 SetCommonVFPActions(VT);
1147 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1148 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1153 for (
MVT VT : F64VecVTs) {
1156 SetCommonVFPActions(VT);
1157 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1158 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1159 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1165 if (!useRVVForFixedLengthVectorVT(VT))
1211 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1238 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1262 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1263 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1300 if (Subtarget.hasStdExtZvkb())
1303 if (Subtarget.hasStdExtZvbb()) {
1327 if (!useRVVForFixedLengthVectorVT(VT))
1348 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1349 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1365 if (Subtarget.hasStdExtZfhmin()) {
1389 if (Subtarget.hasStdExtZfbfmin()) {
1449 if (Subtarget.hasStdExtZfbfmin())
1458 if (Subtarget.hasStdExtA())
1461 if (Subtarget.hasForcedAtomics()) {
1471 if (Subtarget.hasVendorXTHeadMemIdx()) {
1487 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
1513 if (Subtarget.hasStdExtZbb())
1516 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1520 if (Subtarget.hasStdExtZbkb())
1533 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_REVERSE,
1539 if (Subtarget.hasVendorXTHeadMemPair())
1582MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1587bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1589 bool IsScalable)
const {
1596 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1621 unsigned Intrinsic)
const {
1622 auto &
DL =
I.getDataLayout();
1624 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1625 bool IsUnitStrided,
bool UsePtrVal =
false) {
1630 Info.ptrVal =
I.getArgOperand(PtrOp);
1632 Info.fallbackAddressSpace =
1633 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1637 MemTy =
I.getArgOperand(0)->getType();
1640 MemTy =
I.getType();
1650 if (cast<TargetExtType>(MemTy)->
getName() ==
"riscv.vector.tuple")
1653 1 << cast<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))
1655 Info.align =
DL.getABITypeAlign(MemTy);
1665 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1669 switch (Intrinsic) {
1672 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1673 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1674 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1675 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1676 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1677 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1678 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1679 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1680 case Intrinsic::riscv_masked_cmpxchg_i32:
1682 Info.memVT = MVT::i32;
1683 Info.ptrVal =
I.getArgOperand(0);
1689 case Intrinsic::riscv_seg2_load:
1690 case Intrinsic::riscv_seg3_load:
1691 case Intrinsic::riscv_seg4_load:
1692 case Intrinsic::riscv_seg5_load:
1693 case Intrinsic::riscv_seg6_load:
1694 case Intrinsic::riscv_seg7_load:
1695 case Intrinsic::riscv_seg8_load:
1696 return SetRVVLoadStoreInfo( 0,
false,
1698 case Intrinsic::riscv_seg2_store:
1699 case Intrinsic::riscv_seg3_store:
1700 case Intrinsic::riscv_seg4_store:
1701 case Intrinsic::riscv_seg5_store:
1702 case Intrinsic::riscv_seg6_store:
1703 case Intrinsic::riscv_seg7_store:
1704 case Intrinsic::riscv_seg8_store:
1706 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1709 case Intrinsic::riscv_vle:
1710 case Intrinsic::riscv_vle_mask:
1711 case Intrinsic::riscv_vleff:
1712 case Intrinsic::riscv_vleff_mask:
1713 return SetRVVLoadStoreInfo( 1,
1717 case Intrinsic::riscv_vse:
1718 case Intrinsic::riscv_vse_mask:
1719 return SetRVVLoadStoreInfo( 1,
1723 case Intrinsic::riscv_vlse:
1724 case Intrinsic::riscv_vlse_mask:
1725 case Intrinsic::riscv_vloxei:
1726 case Intrinsic::riscv_vloxei_mask:
1727 case Intrinsic::riscv_vluxei:
1728 case Intrinsic::riscv_vluxei_mask:
1729 return SetRVVLoadStoreInfo( 1,
1732 case Intrinsic::riscv_vsse:
1733 case Intrinsic::riscv_vsse_mask:
1734 case Intrinsic::riscv_vsoxei:
1735 case Intrinsic::riscv_vsoxei_mask:
1736 case Intrinsic::riscv_vsuxei:
1737 case Intrinsic::riscv_vsuxei_mask:
1738 return SetRVVLoadStoreInfo( 1,
1741 case Intrinsic::riscv_vlseg2:
1742 case Intrinsic::riscv_vlseg3:
1743 case Intrinsic::riscv_vlseg4:
1744 case Intrinsic::riscv_vlseg5:
1745 case Intrinsic::riscv_vlseg6:
1746 case Intrinsic::riscv_vlseg7:
1747 case Intrinsic::riscv_vlseg8:
1748 case Intrinsic::riscv_vlseg2ff:
1749 case Intrinsic::riscv_vlseg3ff:
1750 case Intrinsic::riscv_vlseg4ff:
1751 case Intrinsic::riscv_vlseg5ff:
1752 case Intrinsic::riscv_vlseg6ff:
1753 case Intrinsic::riscv_vlseg7ff:
1754 case Intrinsic::riscv_vlseg8ff:
1755 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1758 case Intrinsic::riscv_vlseg2_mask:
1759 case Intrinsic::riscv_vlseg3_mask:
1760 case Intrinsic::riscv_vlseg4_mask:
1761 case Intrinsic::riscv_vlseg5_mask:
1762 case Intrinsic::riscv_vlseg6_mask:
1763 case Intrinsic::riscv_vlseg7_mask:
1764 case Intrinsic::riscv_vlseg8_mask:
1765 case Intrinsic::riscv_vlseg2ff_mask:
1766 case Intrinsic::riscv_vlseg3ff_mask:
1767 case Intrinsic::riscv_vlseg4ff_mask:
1768 case Intrinsic::riscv_vlseg5ff_mask:
1769 case Intrinsic::riscv_vlseg6ff_mask:
1770 case Intrinsic::riscv_vlseg7ff_mask:
1771 case Intrinsic::riscv_vlseg8ff_mask:
1772 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1775 case Intrinsic::riscv_vlsseg2:
1776 case Intrinsic::riscv_vlsseg3:
1777 case Intrinsic::riscv_vlsseg4:
1778 case Intrinsic::riscv_vlsseg5:
1779 case Intrinsic::riscv_vlsseg6:
1780 case Intrinsic::riscv_vlsseg7:
1781 case Intrinsic::riscv_vlsseg8:
1782 case Intrinsic::riscv_vloxseg2:
1783 case Intrinsic::riscv_vloxseg3:
1784 case Intrinsic::riscv_vloxseg4:
1785 case Intrinsic::riscv_vloxseg5:
1786 case Intrinsic::riscv_vloxseg6:
1787 case Intrinsic::riscv_vloxseg7:
1788 case Intrinsic::riscv_vloxseg8:
1789 case Intrinsic::riscv_vluxseg2:
1790 case Intrinsic::riscv_vluxseg3:
1791 case Intrinsic::riscv_vluxseg4:
1792 case Intrinsic::riscv_vluxseg5:
1793 case Intrinsic::riscv_vluxseg6:
1794 case Intrinsic::riscv_vluxseg7:
1795 case Intrinsic::riscv_vluxseg8:
1796 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1799 case Intrinsic::riscv_vlsseg2_mask:
1800 case Intrinsic::riscv_vlsseg3_mask:
1801 case Intrinsic::riscv_vlsseg4_mask:
1802 case Intrinsic::riscv_vlsseg5_mask:
1803 case Intrinsic::riscv_vlsseg6_mask:
1804 case Intrinsic::riscv_vlsseg7_mask:
1805 case Intrinsic::riscv_vlsseg8_mask:
1806 case Intrinsic::riscv_vloxseg2_mask:
1807 case Intrinsic::riscv_vloxseg3_mask:
1808 case Intrinsic::riscv_vloxseg4_mask:
1809 case Intrinsic::riscv_vloxseg5_mask:
1810 case Intrinsic::riscv_vloxseg6_mask:
1811 case Intrinsic::riscv_vloxseg7_mask:
1812 case Intrinsic::riscv_vloxseg8_mask:
1813 case Intrinsic::riscv_vluxseg2_mask:
1814 case Intrinsic::riscv_vluxseg3_mask:
1815 case Intrinsic::riscv_vluxseg4_mask:
1816 case Intrinsic::riscv_vluxseg5_mask:
1817 case Intrinsic::riscv_vluxseg6_mask:
1818 case Intrinsic::riscv_vluxseg7_mask:
1819 case Intrinsic::riscv_vluxseg8_mask:
1820 return SetRVVLoadStoreInfo(
I.arg_size() - 6,
1823 case Intrinsic::riscv_vsseg2:
1824 case Intrinsic::riscv_vsseg3:
1825 case Intrinsic::riscv_vsseg4:
1826 case Intrinsic::riscv_vsseg5:
1827 case Intrinsic::riscv_vsseg6:
1828 case Intrinsic::riscv_vsseg7:
1829 case Intrinsic::riscv_vsseg8:
1830 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1833 case Intrinsic::riscv_vsseg2_mask:
1834 case Intrinsic::riscv_vsseg3_mask:
1835 case Intrinsic::riscv_vsseg4_mask:
1836 case Intrinsic::riscv_vsseg5_mask:
1837 case Intrinsic::riscv_vsseg6_mask:
1838 case Intrinsic::riscv_vsseg7_mask:
1839 case Intrinsic::riscv_vsseg8_mask:
1840 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1843 case Intrinsic::riscv_vssseg2:
1844 case Intrinsic::riscv_vssseg3:
1845 case Intrinsic::riscv_vssseg4:
1846 case Intrinsic::riscv_vssseg5:
1847 case Intrinsic::riscv_vssseg6:
1848 case Intrinsic::riscv_vssseg7:
1849 case Intrinsic::riscv_vssseg8:
1850 case Intrinsic::riscv_vsoxseg2:
1851 case Intrinsic::riscv_vsoxseg3:
1852 case Intrinsic::riscv_vsoxseg4:
1853 case Intrinsic::riscv_vsoxseg5:
1854 case Intrinsic::riscv_vsoxseg6:
1855 case Intrinsic::riscv_vsoxseg7:
1856 case Intrinsic::riscv_vsoxseg8:
1857 case Intrinsic::riscv_vsuxseg2:
1858 case Intrinsic::riscv_vsuxseg3:
1859 case Intrinsic::riscv_vsuxseg4:
1860 case Intrinsic::riscv_vsuxseg5:
1861 case Intrinsic::riscv_vsuxseg6:
1862 case Intrinsic::riscv_vsuxseg7:
1863 case Intrinsic::riscv_vsuxseg8:
1864 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1867 case Intrinsic::riscv_vssseg2_mask:
1868 case Intrinsic::riscv_vssseg3_mask:
1869 case Intrinsic::riscv_vssseg4_mask:
1870 case Intrinsic::riscv_vssseg5_mask:
1871 case Intrinsic::riscv_vssseg6_mask:
1872 case Intrinsic::riscv_vssseg7_mask:
1873 case Intrinsic::riscv_vssseg8_mask:
1874 case Intrinsic::riscv_vsoxseg2_mask:
1875 case Intrinsic::riscv_vsoxseg3_mask:
1876 case Intrinsic::riscv_vsoxseg4_mask:
1877 case Intrinsic::riscv_vsoxseg5_mask:
1878 case Intrinsic::riscv_vsoxseg6_mask:
1879 case Intrinsic::riscv_vsoxseg7_mask:
1880 case Intrinsic::riscv_vsoxseg8_mask:
1881 case Intrinsic::riscv_vsuxseg2_mask:
1882 case Intrinsic::riscv_vsuxseg3_mask:
1883 case Intrinsic::riscv_vsuxseg4_mask:
1884 case Intrinsic::riscv_vsuxseg5_mask:
1885 case Intrinsic::riscv_vsuxseg6_mask:
1886 case Intrinsic::riscv_vsuxseg7_mask:
1887 case Intrinsic::riscv_vsuxseg8_mask:
1888 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1929 return isInt<12>(Imm);
1933 return isInt<12>(Imm);
1946 return (SrcBits == 64 && DestBits == 32);
1957 return (SrcBits == 64 && DestBits == 32);
1968 if (SrcBits == DestBits * 2) {
1979 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1980 EVT MemVT = LD->getMemoryVT();
1981 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1991 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1999 return Subtarget.hasStdExtZbb() ||
2000 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
2004 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
2005 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
2016 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
2021 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
2025 EVT VT =
Y.getValueType();
2031 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2032 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
2037 if (Subtarget.hasStdExtZbs())
2038 return X.getValueType().isScalarInteger();
2039 auto *
C = dyn_cast<ConstantSDNode>(
Y);
2041 if (Subtarget.hasVendorXTHeadBs())
2042 return C !=
nullptr;
2044 return C &&
C->getAPIntValue().ule(10);
2064 if (BitSize > Subtarget.
getXLen())
2068 int64_t Val = Imm.getSExtValue();
2076 if (!Subtarget.enableUnalignedScalarMem())
2092 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
2099 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
2103 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2147 if (!Subtarget.hasStdExtZfa())
2150 bool IsSupportedVT =
false;
2151 if (VT == MVT::f16) {
2152 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2153 }
else if (VT == MVT::f32) {
2154 IsSupportedVT =
true;
2155 }
else if (VT == MVT::f64) {
2156 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2157 IsSupportedVT =
true;
2167 bool ForCodeSize)
const {
2168 bool IsLegalVT =
false;
2171 else if (VT == MVT::f32)
2173 else if (VT == MVT::f64)
2175 else if (VT == MVT::bf16)
2176 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2188 return Imm.isZero();
2192 if (Imm.isNegZero())
2197 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2200 Subtarget.
getXLen(), Subtarget);
2206 unsigned Index)
const {
2219 if (EltVT == MVT::i1)
2232 if (Index + ResElts <= MinVLMAX && Index < 31)
2241 return (ResElts * 2) == SrcElts && (Index == 0 || Index == ResElts);
2260 std::optional<MVT> RegisterVT)
const {
2262 if (VT == (Subtarget.
is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2263 *RegisterVT == MVT::Untyped)
2283 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2285 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2300 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2306 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2319 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2320 int64_t
C = RHSC->getSExtValue();
2358 if (VT.
SimpleTy >= MVT::riscv_nxv1i8x2 &&
2359 VT.
SimpleTy <= MVT::riscv_nxv1i8x8)
2361 if (VT.
SimpleTy >= MVT::riscv_nxv2i8x2 &&
2362 VT.
SimpleTy <= MVT::riscv_nxv2i8x8)
2364 if (VT.
SimpleTy >= MVT::riscv_nxv4i8x2 &&
2365 VT.
SimpleTy <= MVT::riscv_nxv4i8x8)
2367 if (VT.
SimpleTy >= MVT::riscv_nxv8i8x2 &&
2368 VT.
SimpleTy <= MVT::riscv_nxv8i8x8)
2370 if (VT.
SimpleTy >= MVT::riscv_nxv16i8x2 &&
2371 VT.
SimpleTy <= MVT::riscv_nxv16i8x4)
2373 if (VT.
SimpleTy == MVT::riscv_nxv32i8x2)
2383 switch (KnownSize) {
2411 return RISCV::VRRegClassID;
2413 return RISCV::VRM2RegClassID;
2415 return RISCV::VRM4RegClassID;
2417 return RISCV::VRM8RegClassID;
2427 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2428 "Unexpected subreg numbering");
2429 return RISCV::sub_vrm1_0 + Index;
2432 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2433 "Unexpected subreg numbering");
2434 return RISCV::sub_vrm2_0 + Index;
2437 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2438 "Unexpected subreg numbering");
2439 return RISCV::sub_vrm4_0 + Index;
2447 unsigned RegsPerField =
2450 switch (RegsPerField) {
2453 return RISCV::VRN2M1RegClassID;
2455 return RISCV::VRN3M1RegClassID;
2457 return RISCV::VRN4M1RegClassID;
2459 return RISCV::VRN5M1RegClassID;
2461 return RISCV::VRN6M1RegClassID;
2463 return RISCV::VRN7M1RegClassID;
2465 return RISCV::VRN8M1RegClassID;
2469 return RISCV::VRN2M2RegClassID;
2471 return RISCV::VRN3M2RegClassID;
2473 return RISCV::VRN4M2RegClassID;
2477 return RISCV::VRN2M4RegClassID;
2485 return RISCV::VRRegClassID;
2494std::pair<unsigned, unsigned>
2496 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2498 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2499 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2500 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2501 "Register classes not ordered");
2508 if (VecRegClassID == SubRegClassID)
2509 return {RISCV::NoSubRegister, 0};
2512 "Only allow scalable vector subvector.");
2514 "Invalid vector tuple insert/extract for vector and subvector with "
2525 unsigned SubRegIdx = RISCV::NoSubRegister;
2526 for (
const unsigned RCID :
2527 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2528 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2532 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2537 return {SubRegIdx, InsertExtractIdx};
2542bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2573unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2580 "Unexpected opcode");
2582 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2584 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2587 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2661bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2662 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2671 "Expected legal fixed length vector!");
2674 unsigned MaxELen = Subtarget.
getELen();
2708 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2715 "Expected to convert into a scalable vector!");
2716 assert(V.getValueType().isFixedLengthVector() &&
2717 "Expected a fixed length vector operand!");
2727 "Expected to convert into a fixed length vector!");
2728 assert(V.getValueType().isScalableVector() &&
2729 "Expected a scalable vector operand!");
2752static std::pair<SDValue, SDValue>
2761static std::pair<SDValue, SDValue>
2774static std::pair<SDValue, SDValue>
2791std::pair<unsigned, unsigned>
2807 return std::make_pair(MinVLMAX, MaxVLMAX);
2819 EVT VT,
unsigned DefinedValues)
const {
2833 std::tie(LMul, Fractional) =
2836 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2838 Cost = (LMul * DLenFactor);
2881 Op.getValueType() == MVT::bf16) {
2882 bool IsStrict =
Op->isStrictFPOpcode();
2887 {Op.getOperand(0), Op.getOperand(1)});
2889 {
Op.getValueType(), MVT::Other},
2895 DAG.
getNode(
Op.getOpcode(),
DL, MVT::f32,
Op.getOperand(0)),
2910 MVT DstVT =
Op.getSimpleValueType();
2911 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2919 Src.getValueType() == MVT::bf16) {
2926 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2934 Opc,
DL, DstVT, Src,
2948 MVT SrcVT = Src.getSimpleValueType();
2954 if (SatVT != DstEltVT)
2957 MVT DstContainerVT = DstVT;
2958 MVT SrcContainerVT = SrcVT;
2964 "Expected same element count");
2973 {Src, Src, DAG.getCondCode(ISD::SETNE),
2974 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2978 if (DstEltSize > (2 * SrcEltSize)) {
2984 MVT CvtContainerVT = DstContainerVT;
2985 MVT CvtEltVT = DstEltVT;
2986 if (SrcEltSize > (2 * DstEltSize)) {
2995 while (CvtContainerVT != DstContainerVT) {
3001 Res = DAG.
getNode(ClipOpc,
DL, CvtContainerVT, Res, Mask, VL);
3008 Res, DAG.
getUNDEF(DstContainerVT), VL);
3018 bool IsStrict =
Op->isStrictFPOpcode();
3019 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
3029 {
Op.getOperand(0), SrcVal});
3030 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
3031 {Ext.getValue(1), Ext.getValue(0)});
3045 case ISD::VP_FROUNDEVEN:
3049 case ISD::VP_FROUNDTOZERO:
3053 case ISD::VP_FFLOOR:
3061 case ISD::VP_FROUND:
3078 MVT VT =
Op.getSimpleValueType();
3085 MVT ContainerVT = VT;
3092 if (
Op->isVPOpcode()) {
3093 Mask =
Op.getOperand(1);
3097 VL =
Op.getOperand(2);
3119 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3133 switch (
Op.getOpcode()) {
3141 case ISD::VP_FFLOOR:
3144 case ISD::VP_FROUND:
3145 case ISD::VP_FROUNDEVEN:
3146 case ISD::VP_FROUNDTOZERO: {
3158 case ISD::VP_FNEARBYINT:
3171 Src, Src, Mask, VL);
3186 MVT VT =
Op.getSimpleValueType();
3190 MVT ContainerVT = VT;
3202 MVT MaskVT = Mask.getSimpleValueType();
3205 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3206 DAG.getUNDEF(MaskVT), Mask, VL});
3210 {Chain, Src, Src, Src, Unorder, VL});
3211 Chain = Src.getValue(1);
3227 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3239 switch (
Op.getOpcode()) {
3250 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3256 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3260 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3269 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3270 Truncated, Mask, VL);
3276 Src, Src, Mask, VL);
3286 MVT VT =
Op.getSimpleValueType();
3314 MVT VT =
Op.getSimpleValueType();
3319 MVT ContainerVT = VT;
3364 "Unexpected vector MVT");
3392 return std::nullopt;
3410 unsigned EltSizeInBits) {
3413 return std::nullopt;
3414 bool IsInteger =
Op.getValueType().isInteger();
3416 std::optional<unsigned> SeqStepDenom;
3417 std::optional<APInt> SeqStepNum;
3418 std::optional<APInt> SeqAddend;
3419 std::optional<std::pair<APInt, unsigned>> PrevElt;
3420 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3425 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3427 if (Elt.isUndef()) {
3428 Elts[
Idx] = std::nullopt;
3432 Elts[
Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
3437 return std::nullopt;
3438 Elts[
Idx] = *ExactInteger;
3451 unsigned IdxDiff =
Idx - PrevElt->second;
3452 APInt ValDiff = *Elt - PrevElt->first;
3460 int64_t Remainder = ValDiff.
srem(IdxDiff);
3465 return std::nullopt;
3466 ValDiff = ValDiff.
sdiv(IdxDiff);
3471 SeqStepNum = ValDiff;
3472 else if (ValDiff != SeqStepNum)
3473 return std::nullopt;
3476 SeqStepDenom = IdxDiff;
3477 else if (IdxDiff != *SeqStepDenom)
3478 return std::nullopt;
3482 if (!PrevElt || PrevElt->first != *Elt)
3483 PrevElt = std::make_pair(*Elt,
Idx);
3487 if (!SeqStepNum || !SeqStepDenom)
3488 return std::nullopt;
3496 (
APInt(EltSizeInBits,
Idx,
false,
true) *
3498 .sdiv(*SeqStepDenom);
3500 APInt Addend = *Elt - ExpectedVal;
3503 else if (Addend != SeqAddend)
3504 return std::nullopt;
3507 assert(SeqAddend &&
"Must have an addend if we have a step");
3509 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
3510 SeqAddend->getSExtValue()};
3525 MVT SrcVT = Src.getSimpleValueType();
3535 auto *CIdx = dyn_cast<ConstantSDNode>(
Idx);
3541 MVT ContainerVT = VT;
3545 MVT SrcContainerVT = SrcVT;
3580 MVT VT =
Op.getSimpleValueType();
3592 unsigned MostCommonCount = 0;
3594 unsigned NumUndefElts =
3602 unsigned NumScalarLoads = 0;
3608 unsigned &Count = ValueCounts[V];
3610 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3611 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3616 if (++Count >= MostCommonCount) {
3618 MostCommonCount = Count;
3622 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3623 unsigned NumDefElts = NumElts - NumUndefElts;
3624 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3630 ((MostCommonCount > DominantValueCountThreshold) ||
3643 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3644 LastOp != DominantValue) {
3653 Processed.insert(LastOp);
3658 const SDValue &V = OpIdx.value();
3659 if (V.isUndef() || !Processed.insert(V).second)
3661 if (ValueCounts[V] == 1) {
3670 return DAG.getConstant(V == V1, DL, XLenVT);
3686 MVT VT =
Op.getSimpleValueType();
3716 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3717 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3725 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3726 MVT IntegerViaVecVT =
3731 unsigned BitPos = 0, IntegerEltIdx = 0;
3734 for (
unsigned I = 0;
I < NumElts;) {
3736 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3737 Bits |= ((
uint64_t)BitValue << BitPos);
3743 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3744 if (NumViaIntegerBits <= 32)
3745 Bits = SignExtend64<32>(Bits);
3747 Elts[IntegerEltIdx] = Elt;
3756 if (NumElts < NumViaIntegerBits) {
3760 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3788 int64_t StepNumerator = SimpleVID->StepNumerator;
3789 unsigned StepDenominator = SimpleVID->StepDenominator;
3790 int64_t Addend = SimpleVID->Addend;
3792 assert(StepNumerator != 0 &&
"Invalid step");
3793 bool Negate =
false;
3794 int64_t SplatStepVal = StepNumerator;
3798 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3800 Negate = StepNumerator < 0;
3802 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3809 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3810 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3812 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3815 MVT VIDContainerVT =
3823 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3824 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3826 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3828 if (StepDenominator != 1) {
3833 if (Addend != 0 || Negate) {
3852 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3853 "Unexpected sequence type");
3857 unsigned ViaVecLen =
3861 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3864 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3865 const auto &SeqV = OpIdx.value();
3866 if (!SeqV.isUndef())
3868 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3874 if (ViaIntVT == MVT::i32)
3875 SplatValue = SignExtend64<32>(SplatValue);
3897 const auto *BV = cast<BuildVectorSDNode>(
Op);
3900 BV->getRepeatedSequence(Sequence) &&
3901 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3902 unsigned SeqLen = Sequence.size();
3904 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3905 ViaIntVT == MVT::i64) &&
3906 "Unexpected sequence type");
3911 const unsigned RequiredVL = NumElts / SeqLen;
3912 const unsigned ViaVecLen =
3914 NumElts : RequiredVL;
3917 unsigned EltIdx = 0;
3918 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3922 for (
const auto &SeqV : Sequence) {
3923 if (!SeqV.isUndef())
3925 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3932 if (ViaIntVT == MVT::i32)
3933 SplatValue = SignExtend64<32>(SplatValue);
3939 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3940 "Unexpected bitcast sequence");
3941 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3944 MVT ViaContainerVT =
3951 if (ViaVecLen != RequiredVL)
3971 Source, DAG, Subtarget);
3991 return RISCV::PACKH;
3993 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
4008 MVT VT =
Op.getSimpleValueType();
4016 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4021 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
4035 if (Subtarget.hasStdExtZbkb())
4040 ElemDL, XLenVT,
A,
B),
4052 NewOperands.
reserve(NumElts / 2);
4054 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
4064 MVT VT =
Op.getSimpleValueType();
4075 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) || EltVT == MVT::bf16) {
4080 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4081 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4084 if (
auto *
C = dyn_cast<ConstantFPSDNode>(Elem)) {
4168 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
4172 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4195 unsigned NumUndefElts =
4197 unsigned NumDefElts = NumElts - NumUndefElts;
4198 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4205 for (
unsigned i = 0; i < NumElts; i++) {
4207 if (i < NumElts / 2) {
4214 bool SelectMaskVal = (i < NumElts / 2);
4217 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4218 MaskVals.
size() == NumElts);
4253 unsigned UndefCount = 0;
4260 LinearBudget -= PerSlideCost;
4263 LinearBudget -= PerSlideCost;
4266 LinearBudget -= PerSlideCost;
4269 if (LinearBudget < 0)
4274 "Illegal type which will result in reserved encoding");
4299 Vec,
Offset, Mask, VL, Policy);
4312 Vec,
Offset, Mask, VL, Policy);
4322 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4323 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4324 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4327 if ((LoC >> 31) == HiC)
4338 (isa<RegisterSDNode>(VL) &&
4339 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4341 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4356 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4357 Hi.getConstantOperandVal(1) == 31)
4376 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4388 bool HasPassthru = Passthru && !Passthru.
isUndef();
4389 if (!HasPassthru && !Passthru)
4396 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
4397 EltVT == MVT::bf16) {
4398 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4399 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
4413 if (Scalar.getValueType().bitsLE(XLenVT)) {
4420 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4424 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4425 "Unexpected scalar for splat lowering!");
4449 SDValue ExtractedVal = Scalar.getOperand(0);
4454 MVT ExtractedContainerVT = ExtractedVT;
4457 DAG, ExtractedContainerVT, Subtarget);
4459 ExtractedVal, DAG, Subtarget);
4461 if (ExtractedContainerVT.
bitsLE(VT))
4476 if (!Scalar.getValueType().bitsLE(XLenVT))
4479 VT,
DL, DAG, Subtarget);
4487 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4507 if (Src != V2.getOperand(0))
4512 if (Src.getValueType().getVectorNumElements() != (NumElts * 2))
4517 V2.getConstantOperandVal(1) != NumElts)
4535 int Size = Mask.size();
4537 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4543 EvenSrc = StartIndexes[0];
4544 OddSrc = StartIndexes[1];
4547 if (EvenSrc != 0 && OddSrc != 0)
4557 int HalfNumElts = NumElts / 2;
4558 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4574 int Size = Mask.size();
4586 for (
int i = 0; i !=
Size; ++i) {
4592 int StartIdx = i - (M %
Size);
4600 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4603 Rotation = CandidateRotation;
4604 else if (Rotation != CandidateRotation)
4609 int MaskSrc = M <
Size ? 0 : 1;
4614 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4619 TargetSrc = MaskSrc;
4620 else if (TargetSrc != MaskSrc)
4627 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4628 assert((LoSrc >= 0 || HiSrc >= 0) &&
4629 "Failed to find a rotated input vector!");
4643 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
4650 unsigned Shift = Index * EltBits;
4677 auto findNonEXTRACT_SUBVECTORParent =
4678 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4683 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4684 Offset += Parent.getConstantOperandVal(1);
4685 Parent = Parent.getOperand(0);
4687 return std::make_pair(Parent,
Offset);
4690 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4691 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4700 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4701 if (NewMask[i] == -1)
4704 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4705 NewMask[i] = NewMask[i] + V1IndexOffset;
4709 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4715 if (NewMask[0] <= 0)
4719 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4720 if (NewMask[i - 1] + 1 != NewMask[i])
4724 MVT SrcVT = Src.getSimpleValueType();
4755 int NumSubElts, Index;
4760 bool OpsSwapped = Mask[Index] < (int)NumElts;
4761 SDValue InPlace = OpsSwapped ? V2 : V1;
4762 SDValue ToInsert = OpsSwapped ? V1 : V2;
4772 if (NumSubElts + Index >= (
int)NumElts)
4786 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4798 bool OpsSwapped =
false;
4799 if (!isa<BuildVectorSDNode>(V1)) {
4800 if (!isa<BuildVectorSDNode>(V2))
4805 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4813 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4814 for (
unsigned i = S; i != E; ++i)
4815 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4821 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4822 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4825 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4827 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4850 auto OpCode = IsVSlidedown ?
4855 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4858 Splat, TrueMask, VL);
4869 for (
unsigned i = 0; i < Mask.size(); i++)
4870 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
4873 for (
unsigned i = 0; i < Factor; i++) {
4884 for (
unsigned i = 0; i < Mask.size() / Factor; i++) {
4885 unsigned j = i * Factor + Index;
4886 if (Mask[j] != -1 && (
unsigned)Mask[j] != i)
4899 MVT VT = V.getSimpleValueType();
4914 EC.multiplyCoefficientBy(Factor));
4933 MVT VecContainerVT = VecVT;
4950 MVT WideContainerVT = WideVT;
4956 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4963 if (Subtarget.hasStdExtZvbb()) {
4968 OffsetVec, Passthru, Mask, VL);
4970 Interleaved, EvenV, Passthru, Mask, VL);
4978 OddV, Passthru, Mask, VL);
4984 OddV, AllOnesVec, Passthru, Mask, VL);
4992 Interleaved, OddsMul, Passthru, Mask, VL);
4999 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
5045 if (ViaEltSize > NumElts)
5054 if (ViaEltSize > NumElts)
5060 if (ViaEltSize > NumElts)
5069 MVT &RotateVT,
unsigned &RotateAmt) {
5075 unsigned NumSubElts;
5077 NumElts, NumSubElts, RotateAmt))
5080 NumElts / NumSubElts);
5148 unsigned NumOfSrcRegs = NumElts / NumOpElts;
5149 unsigned NumOfDestRegs = NumElts / NumOpElts;
5158 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs,
5159 [&]() {
Operands.emplace_back(); },
5160 [&](
ArrayRef<int> SrcSubMask,
unsigned SrcVecIdx,
unsigned DstVecIdx) {
5161 Operands.emplace_back().emplace_back(
5162 SrcVecIdx, UINT_MAX,
5165 [&](
ArrayRef<int> SrcSubMask,
unsigned Idx1,
unsigned Idx2,
bool NewReg) {
5171 assert(
Operands.size() == NumOfDestRegs &&
"Whole vector must be processed");
5176 unsigned NumShuffles = std::accumulate(
5183 for (const auto &P : Data) {
5184 unsigned Idx2 = std::get<1>(P);
5185 ArrayRef<int> Mask = std::get<2>(P);
5186 if (Idx2 != UINT_MAX)
5188 else if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
5193 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
5194 (NumOfDestRegs <= 2 && NumShuffles >= 4))
5196 auto ExtractValue = [&, &DAG = DAG](
SDValue SrcVec,
unsigned ExtractIdx) {
5198 DAG.getVectorIdxConstant(ExtractIdx,
DL));
5202 auto PerformShuffle = [&, &DAG = DAG](
SDValue SubVec1,
SDValue SubVec2,
5204 SDValue SubVec = DAG.getVectorShuffle(OneRegVT,
DL, SubVec1, SubVec2, Mask);
5207 SDValue Vec = DAG.getUNDEF(ContainerVT);
5212 for (
unsigned I : seq<unsigned>(
Data.size())) {
5213 const auto &[Idx1, Idx2,
_] =
Data[
I];
5216 "Expected both indices to be extracted already.");
5219 SDValue V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
5220 (Idx1 % NumOfSrcRegs) * NumOpElts);
5222 if (Idx2 != UINT_MAX)
5223 Values[Idx2] = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
5224 (Idx2 % NumOfSrcRegs) * NumOpElts);
5227 for (
const auto &[Idx1, Idx2, Mask] :
Data) {
5229 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.
at(Idx2);
5230 V = PerformShuffle(V1, V2, Mask);
5234 unsigned InsertIdx =
I * NumOpElts;
5237 DAG.getVectorIdxConstant(InsertIdx,
DL));
5247 bool SawUndef =
false;
5248 for (
unsigned i = 0; i < Mask.size(); i++) {
5249 if (Mask[i] == -1) {
5255 if (i > (
unsigned)Mask[i])
5257 if (Mask[i] <=
Last)
5288 for (
int Idx : Mask) {
5291 unsigned SrcIdx =
Idx % Mask.size();
5293 if (Srcs[SrcIdx] == -1)
5296 else if (Srcs[SrcIdx] != Src)
5302 for (
int Lane : Srcs) {
5315 for (
unsigned I = 0;
I < Mask.size();
I++) {
5319 NewMask[
I] = Mask[
I] % Mask.size();
5333 MVT VT =
Op.getSimpleValueType();
5341 if (ElementSize > 32)
5364 MVT VT =
Op.getSimpleValueType();
5379 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5403 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5404 V = V.getOperand(
Offset / OpElements);
5410 auto *Ld = cast<LoadSDNode>(V);
5420 SDValue Ops[] = {Ld->getChain(),
5434 MVT SplatVT = ContainerVT;
5437 if (SVT == MVT::bf16 ||
5438 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
5447 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5448 Ld->getPointerInfo().getWithOffset(
Offset),
5449 Ld->getOriginalAlign(),
5453 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5454 Ld->getOriginalAlign(),
5455 Ld->getMemOperand()->getFlags());
5467 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5470 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5492 if (Subtarget.hasStdExtZvkb())
5503 LoV = LoSrc == 0 ? V1 : V2;
5507 HiV = HiSrc == 0 ? V1 : V2;
5513 unsigned InvRotate = NumElts - Rotation;
5523 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5541 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
5542 for (
unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
5558 int EvenSrc, OddSrc;
5566 bool LaneIsUndef[2] = {
true,
true};
5567 for (
unsigned i = 0; i < Mask.size(); i++)
5568 LaneIsUndef[i % 2] &= (Mask[i] == -1);
5570 int Size = Mask.size();
5572 if (LaneIsUndef[0]) {
5575 assert(EvenSrc >= 0 &&
"Undef source?");
5576 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5581 if (LaneIsUndef[1]) {
5584 assert(OddSrc >= 0 &&
"Undef source?");
5585 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5595 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5615 for (
auto Idx : Mask) {
5631 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
5632 for (
unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
5645 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5674 MVT IndexContainerVT =
5679 for (
int MaskIndex : Mask) {
5680 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5689 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5699 for (
int MaskIndex : Mask) {
5700 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5701 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5703 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5734 for (
int MaskIndex : Mask) {
5735 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5739 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5771RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5773 MVT VT =
Op.getSimpleValueType();
5777 MVT ContainerVT = VT;
5780 if (
Op->isVPOpcode()) {
5781 Mask =
Op.getOperand(1);
5785 VL =
Op.getOperand(2);
5791 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5793 FloatEltVT = MVT::f32;
5800 "Expected legal float type!");
5807 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5810 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5815 if (FloatVT.
bitsGT(VT)) {
5816 if (
Op->isVPOpcode())
5817 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5826 if (!
Op->isVPOpcode())
5830 MVT ContainerFloatVT =
5833 Src, Mask, RTZRM, VL);
5840 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5844 if (
Op->isVPOpcode()) {
5853 else if (IntVT.
bitsGT(VT))
5858 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5863 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5864 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5869 unsigned Adjust = ExponentBias + (EltSize - 1);
5871 if (
Op->isVPOpcode())
5881 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5882 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5892 MVT SrcVT =
Source.getSimpleValueType();
5901 SrcVT = ContainerVT;
5914 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5931 auto *
Load = cast<LoadSDNode>(
Op);
5932 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5935 Load->getMemoryVT(),
5936 *
Load->getMemOperand()))
5940 MVT VT =
Op.getSimpleValueType();
5942 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5943 "Unexpected unaligned RVV load type");
5947 "Expecting equally-sized RVV vector types to be legal");
5949 Load->getPointerInfo(),
Load->getOriginalAlign(),
5950 Load->getMemOperand()->getFlags());
5960 auto *
Store = cast<StoreSDNode>(
Op);
5961 assert(Store &&
Store->getValue().getValueType().isVector() &&
5962 "Expected vector store");
5965 Store->getMemoryVT(),
5966 *
Store->getMemOperand()))
5973 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5974 "Unexpected unaligned RVV store type");
5978 "Expecting equally-sized RVV vector types to be legal");
5979 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5981 Store->getPointerInfo(),
Store->getOriginalAlign(),
5982 Store->getMemOperand()->getFlags());
5987 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5989 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
6016 unsigned ShiftAmt, AddOpc;
6027 MVT VT =
Op.getSimpleValueType();
6028 const APFloat &
Imm = cast<ConstantFPSDNode>(
Op)->getValueAPF();
6031 bool Negate =
false;
6035 if (Index < 0 &&
Imm.isNegative()) {
6063 if (Subtarget.hasStdExtZtso()) {
6087 MVT VT =
Op.getSimpleValueType();
6089 unsigned Check =
Op.getConstantOperandVal(1);
6090 unsigned TDCMask = 0;
6118 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
6123 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
6125 VL =
Op.getOperand(3);
6128 VL,
Op->getFlags());
6143 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
6145 MVT MaskContainerVT =
6148 VL =
Op.getOperand(3);
6153 Mask, VL,
Op->getFlags());
6156 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
6161 DAG.
getUNDEF(ContainerVT), Mask, VL});
6165 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
6169 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
6173 DAG.
getUNDEF(ContainerVT), Mask, VL});
6189 MVT VT =
Op.getSimpleValueType();
6216 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
6223 MVT ContainerVT = VT;
6231 if (
Op->isVPOpcode()) {
6232 Mask =
Op.getOperand(2);
6236 VL =
Op.getOperand(3);
6244 {X, X, DAG.getCondCode(ISD::SETOEQ),
6245 DAG.getUNDEF(ContainerVT), Mask, VL});
6253 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
6254 DAG.getUNDEF(ContainerVT), Mask, VL});
6264 DAG.
getUNDEF(ContainerVT), Mask, VL);
6274 "Wrong opcode for lowering FABS or FNEG.");
6277 MVT VT =
Op.getSimpleValueType();
6278 assert((VT == MVT::f16 || VT == MVT::bf16) &&
"Unexpected type");
6285 Mask = Mask.sext(Subtarget.
getXLen());
6298 MVT VT =
Op.getSimpleValueType();
6299 assert((VT == MVT::f16 || VT == MVT::bf16) &&
"Unexpected type");
6309 if (SignSize == Subtarget.
getXLen()) {
6311 }
else if (SignSize == 16) {
6313 }
else if (SignSize == 32) {
6315 }
else if (SignSize == 64) {
6316 assert(XLenVT == MVT::i32 &&
"Unexpected type");
6326 if (ShiftAmount > 0) {
6329 }
else if (ShiftAmount < 0) {
6355#define OP_CASE(NODE) \
6357 return RISCVISD::NODE##_VL;
6358#define VP_CASE(NODE) \
6359 case ISD::VP_##NODE: \
6360 return RISCVISD::NODE##_VL;
6362 switch (
Op.getOpcode()) {
6440 case ISD::VP_CTLZ_ZERO_UNDEF:
6443 case ISD::VP_CTTZ_ZERO_UNDEF:
6452 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6457 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6462 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6465 case ISD::VP_SELECT:
6474 case ISD::VP_SIGN_EXTEND:
6476 case ISD::VP_ZERO_EXTEND:
6478 case ISD::VP_FP_TO_SINT:
6480 case ISD::VP_FP_TO_UINT:
6483 case ISD::VP_FMINNUM:
6486 case ISD::VP_FMAXNUM:
6491 case ISD::VP_LLRINT:
6503 "not a RISC-V target specific op");
6507 "adding target specific op should update this function");
6527 "not a RISC-V target specific op");
6531 "adding target specific op should update this function");
6544 if (
Op.getValueType() == MVT::nxv32f16 &&
6548 if (
Op.getValueType() == MVT::nxv32bf16)
6561 if (!
Op.getOperand(j).getValueType().isVector()) {
6562 LoOperands[j] =
Op.getOperand(j);
6563 HiOperands[j] =
Op.getOperand(j);
6566 std::tie(LoOperands[j], HiOperands[j]) =
6571 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6573 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6588 std::tie(LoOperands[j], HiOperands[j]) =
6592 if (!
Op.getOperand(j).getValueType().isVector()) {
6593 LoOperands[j] =
Op.getOperand(j);
6594 HiOperands[j] =
Op.getOperand(j);
6597 std::tie(LoOperands[j], HiOperands[j]) =
6602 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6604 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6614 auto [EVLLo, EVLHi] =
6615 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6619 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6621 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6639 if (!
Op.getOperand(j).getValueType().isVector()) {
6640 LoOperands[j] =
Op.getOperand(j);
6641 HiOperands[j] =
Op.getOperand(j);
6644 std::tie(LoOperands[j], HiOperands[j]) =
6649 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6652 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6661 switch (
Op.getOpcode()) {
6667 return lowerGlobalAddress(
Op, DAG);
6669 return lowerBlockAddress(
Op, DAG);
6671 return lowerConstantPool(
Op, DAG);
6673 return lowerJumpTable(
Op, DAG);
6675 return lowerGlobalTLSAddress(
Op, DAG);
6679 return lowerConstantFP(
Op, DAG);
6681 return lowerSELECT(
Op, DAG);
6683 return lowerBRCOND(
Op, DAG);
6685 return lowerVASTART(
Op, DAG);
6687 return lowerFRAMEADDR(
Op, DAG);
6689 return lowerRETURNADDR(
Op, DAG);
6691 return lowerShiftLeftParts(
Op, DAG);
6693 return lowerShiftRightParts(
Op, DAG,
true);
6695 return lowerShiftRightParts(
Op, DAG,
false);
6698 if (
Op.getValueType().isFixedLengthVector()) {
6699 assert(Subtarget.hasStdExtZvkb());
6700 return lowerToScalableOp(
Op, DAG);
6702 assert(Subtarget.hasVendorXTHeadBb() &&
6703 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6704 "Unexpected custom legalization");
6706 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6711 EVT VT =
Op.getValueType();
6715 if (Op0VT == MVT::i16 &&
6717 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
6721 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6726 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.
is64Bit() &&
6742 "Unexpected types");
6776 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6778 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6780 return LowerINTRINSIC_VOID(
Op, DAG);
6782 return LowerIS_FPCLASS(
Op, DAG);
6784 MVT VT =
Op.getSimpleValueType();
6786 assert(Subtarget.hasStdExtZvbb());
6787 return lowerToScalableOp(
Op, DAG);
6790 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6800 if (!
Op.getSimpleValueType().isVector())
6802 return lowerVectorTruncLike(
Op, DAG);
6805 if (
Op.getOperand(0).getValueType().isVector() &&
6806 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6807 return lowerVectorMaskExt(
Op, DAG, 1);
6810 if (
Op.getOperand(0).getValueType().isVector() &&
6811 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6812 return lowerVectorMaskExt(
Op, DAG, -1);
6815 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6817 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6819 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6821 MVT VT =
Op.getSimpleValueType();
6829 MVT ContainerVT = VT;
6837 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6841 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6849 MVT VT =
Op.getSimpleValueType();
6869 }
else if ((Val % 8) == 0) {
6885 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6886 Op.getOperand(1).getValueType() == MVT::i32) {
6903 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6906 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6909 if (
Op.getValueType().isVector() &&
6910 ((
Op.getValueType().getScalarType() == MVT::f16 &&
6913 Op.getValueType().getScalarType() == MVT::bf16)) {
6929 Op1.getValueType().isVector() &&
6930 ((Op1.getValueType().getScalarType() == MVT::f16 &&
6933 Op1.getValueType().getScalarType() == MVT::bf16)) {
6939 Op1.getValueType().getVectorElementCount());
6942 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6952 MVT VT =
Op.getSimpleValueType();
6955 bool IsStrict =
Op->isStrictFPOpcode();
6956 SDValue Src =
Op.getOperand(0 + IsStrict);
6957 MVT SrcVT = Src.getSimpleValueType();
6968 "Unexpected vector element types");
6972 if (EltSize > (2 * SrcEltSize)) {
6984 Op.getOperand(0), Ext);
6988 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6993 auto [FExt, Chain] =
6995 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
7002 if (SrcEltSize > (2 * EltSize)) {
7005 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
7010 Op.getOperand(0), Src);
7025 Op.getOperand(0), Src);
7039 unsigned RVVOpc = 0;
7040 switch (
Op.getOpcode()) {
7072 "Expected same element count");
7079 Op.getOperand(0), Src, Mask, VL);
7083 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
7098 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
7105 MVT VT =
Op.getSimpleValueType();
7125 bool IsStrict =
Op->isStrictFPOpcode();
7126 SDValue Op0 = IsStrict ?
Op.getOperand(1) :
Op.getOperand(0);
7130 std::tie(Res, Chain) =
7131 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions,
DL, Chain);
7146 bool IsStrict =
Op->isStrictFPOpcode();
7147 SDValue Op0 = IsStrict ?
Op.getOperand(1) :
Op.getOperand(0);
7153 std::tie(Res, Chain) =
makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
7154 CallOptions,
DL, Chain);
7171 if (
Op.getValueType().isVector())
7176 assert(
Op.getOperand(0).getValueType() == MVT::f16 &&
7177 "Unexpected custom legalisation");
7180 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), Ext);
7186 assert(
Op.getOperand(1).getValueType() == MVT::f16 &&
7187 "Unexpected custom legalisation");
7190 {
Op.getOperand(0),
Op.getOperand(1)});
7191 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
7192 {Ext.getValue(1), Ext.getValue(0)});
7199 return lowerVECREDUCE(
Op, DAG);
7203 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
7204 return lowerVectorMaskVecReduction(
Op, DAG,
false);
7205 return lowerVECREDUCE(
Op, DAG);
7212 return lowerFPVECREDUCE(
Op, DAG);
7213 case ISD::VP_REDUCE_ADD:
7214 case ISD::VP_REDUCE_UMAX:
7215 case ISD::VP_REDUCE_SMAX:
7216 case ISD::VP_REDUCE_UMIN:
7217 case ISD::VP_REDUCE_SMIN:
7218 case ISD::VP_REDUCE_FADD:
7219 case ISD::VP_REDUCE_SEQ_FADD:
7220 case ISD::VP_REDUCE_FMIN:
7221 case ISD::VP_REDUCE_FMAX:
7222 case ISD::VP_REDUCE_FMINIMUM:
7223 case ISD::VP_REDUCE_FMAXIMUM:
7226 return lowerVPREDUCE(
Op, DAG);
7227 case ISD::VP_REDUCE_AND:
7228 case ISD::VP_REDUCE_OR:
7229 case ISD::VP_REDUCE_XOR:
7230 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
7231 return lowerVectorMaskVecReduction(
Op, DAG,
true);
7232 return lowerVPREDUCE(
Op, DAG);
7233 case ISD::VP_CTTZ_ELTS:
7234 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7235 return lowerVPCttzElements(
Op, DAG);
7239 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
7242 return lowerINSERT_SUBVECTOR(
Op, DAG);
7244 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
7246 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
7248 return lowerVECTOR_INTERLEAVE(
Op, DAG);
7250 return lowerSTEP_VECTOR(
Op, DAG);
7252 return lowerVECTOR_REVERSE(
Op, DAG);
7254 return lowerVECTOR_SPLICE(
Op, DAG);
7258 MVT VT =
Op.getSimpleValueType();
7260 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
7261 EltVT == MVT::bf16) {
7264 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
7265 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
7275 if (EltVT == MVT::i1)
7276 return lowerVectorMaskSplat(
Op, DAG);
7285 MVT VT =
Op.getSimpleValueType();
7286 MVT ContainerVT = VT;
7304 Op->ops().take_front(HalfNumOps));
7306 Op->ops().drop_front(HalfNumOps));
7310 unsigned NumOpElts =
7311 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
7314 SDValue SubVec = OpIdx.value();
7325 auto *Load = cast<LoadSDNode>(
Op);
7326 EVT VecTy = Load->getMemoryVT();
7333 unsigned NumElts = Sz / (NF * 8);
7334 int Log2LMUL =
Log2_64(NumElts) - 3;
7337 Flag.setNoUnsignedWrap(
true);
7339 SDValue BasePtr = Load->getBasePtr();
7347 for (
unsigned i = 0; i < NF; ++i) {
7360 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
7362 if (
Op.getValueType().isFixedLengthVector())
7363 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
7367 auto *Store = cast<StoreSDNode>(
Op);
7368 SDValue StoredVal = Store->getValue();
7376 unsigned NumElts = Sz / (NF * 8);
7377 int Log2LMUL =
Log2_64(NumElts) - 3;
7380 Flag.setNoUnsignedWrap(
true);
7382 SDValue Chain = Store->getChain();
7383 SDValue BasePtr = Store->getBasePtr();
7390 for (
unsigned i = 0; i < NF; ++i) {
7394 Ret = DAG.
getStore(Chain,
DL, Extract, BasePtr,
7396 Store->getOriginalAlign(),
7397 Store->getMemOperand()->getFlags());
7398 Chain = Ret.getValue(0);
7404 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
7406 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
7407 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
7412 return lowerMaskedLoad(
Op, DAG);
7415 return lowerMaskedStore(
Op, DAG);
7417 return lowerVectorCompress(
Op, DAG);
7426 EVT VT =
Op.getValueType();
7437 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
7439 MVT VT =
Op.getSimpleValueType();
7444 "Unexpected CondCode");
7452 if (isa<ConstantSDNode>(
RHS)) {
7453 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
7454 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
7476 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
7492 return lowerToScalableOp(
Op, DAG);
7496 if (
Op.getSimpleValueType().isFixedLengthVector())
7497 return lowerToScalableOp(
Op, DAG);
7499 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
7500 "Unexpected custom legalisation");
7504 if (
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16)
7530 return lowerToScalableOp(
Op, DAG);
7534 EVT VT =
Op->getValueType(0);
7549 return lowerABS(
Op, DAG);
7554 if (Subtarget.hasStdExtZvbb())
7555 return lowerToScalableOp(
Op, DAG);
7557 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7559 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
7561 if (
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16)
7565 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
7574 return lowerToScalableOp(
Op, DAG);
7577 return lowerVectorStrictFSetcc(
Op, DAG);
7587 case ISD::VP_GATHER:
7588 return lowerMaskedGather(
Op, DAG);
7590 case ISD::VP_SCATTER:
7591 return lowerMaskedScatter(
Op, DAG);
7593 return lowerGET_ROUNDING(
Op, DAG);
7595 return lowerSET_ROUNDING(
Op, DAG);
7597 return lowerEH_DWARF_CFA(
Op, DAG);
7599 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7600 return lowerVPMergeMask(
Op, DAG);
7602 case ISD::VP_SELECT:
7610 case ISD::VP_UADDSAT:
7611 case ISD::VP_USUBSAT:
7612 case ISD::VP_SADDSAT:
7613 case ISD::VP_SSUBSAT:
7615 case ISD::VP_LLRINT:
7616 return lowerVPOp(
Op, DAG);
7620 return lowerLogicVPOp(
Op, DAG);
7629 case ISD::VP_FMINNUM:
7630 case ISD::VP_FMAXNUM:
7631 case ISD::VP_FCOPYSIGN:
7638 return lowerVPOp(
Op, DAG);
7639 case ISD::VP_IS_FPCLASS:
7640 return LowerIS_FPCLASS(
Op, DAG);
7641 case ISD::VP_SIGN_EXTEND:
7642 case ISD::VP_ZERO_EXTEND:
7643 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7644 return lowerVPExtMaskOp(
Op, DAG);
7645 return lowerVPOp(
Op, DAG);
7646 case ISD::VP_TRUNCATE:
7647 return lowerVectorTruncLike(
Op, DAG);
7648 case ISD::VP_FP_EXTEND:
7649 case ISD::VP_FP_ROUND:
7650 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7651 case ISD::VP_SINT_TO_FP:
7652 case ISD::VP_UINT_TO_FP:
7653 if (
Op.getValueType().isVector() &&
7654 ((
Op.getValueType().getScalarType() == MVT::f16 &&
7657 Op.getValueType().getScalarType() == MVT::bf16)) {
7670 case ISD::VP_FP_TO_SINT:
7671 case ISD::VP_FP_TO_UINT:
7673 Op1.getValueType().isVector() &&
7674 ((Op1.getValueType().getScalarType() == MVT::f16 &&
7677 Op1.getValueType().getScalarType() == MVT::bf16)) {
7683 Op1.getValueType().getVectorElementCount());
7687 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7689 return lowerVPFPIntConvOp(
Op, DAG);
7693 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7694 return lowerVPSetCCMaskOp(
Op, DAG);
7700 case ISD::VP_BITREVERSE:
7702 return lowerVPOp(
Op, DAG);
7704 case ISD::VP_CTLZ_ZERO_UNDEF:
7705 if (Subtarget.hasStdExtZvbb())
7706 return lowerVPOp(
Op, DAG);
7707 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7709 case ISD::VP_CTTZ_ZERO_UNDEF:
7710 if (Subtarget.hasStdExtZvbb())
7711 return lowerVPOp(
Op, DAG);
7712 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7714 return lowerVPOp(
Op, DAG);
7715 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7716 return lowerVPStridedLoad(
Op, DAG);
7717 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7718 return lowerVPStridedStore(
Op, DAG);
7720 case ISD::VP_FFLOOR:
7722 case ISD::VP_FNEARBYINT:
7723 case ISD::VP_FROUND:
7724 case ISD::VP_FROUNDEVEN:
7725 case ISD::VP_FROUNDTOZERO:
7729 case ISD::VP_FMAXIMUM:
7730 case ISD::VP_FMINIMUM:
7734 case ISD::EXPERIMENTAL_VP_SPLICE:
7735 return lowerVPSpliceExperimental(
Op, DAG);
7736 case ISD::EXPERIMENTAL_VP_REVERSE:
7737 return lowerVPReverseExperimental(
Op, DAG);
7738 case ISD::EXPERIMENTAL_VP_SPLAT:
7739 return lowerVPSplatExperimental(
Op, DAG);
7742 "llvm.clear_cache only needs custom lower on Linux targets");
7745 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7746 Op.getOperand(2), Flags,
DL);
7749 return lowerDYNAMIC_STACKALLOC(
Op, DAG);
7751 return lowerINIT_TRAMPOLINE(
Op, DAG);
7753 return lowerADJUST_TRAMPOLINE(
Op, DAG);
7760 MakeLibCallOptions CallOptions;
7761 std::pair<SDValue, SDValue> CallResult =
7762 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7763 {Start,
End, Flags}, CallOptions,
DL, InChain);
7766 return CallResult.second;
7779 std::unique_ptr<MCCodeEmitter> CodeEmitter(
7786 const Value *TrmpAddr = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
7798 constexpr unsigned StaticChainOffset = 16;
7799 constexpr unsigned FunctionAddressOffset = 24;
7803 auto GetEncoding = [&](
const MCInst &MC) {
7806 CodeEmitter->encodeInstruction(MC, CB, Fixups, *STI);
7816 GetEncoding(
MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X7).addImm(0)),
7821 MCInstBuilder(RISCV::LD).addReg(RISCV::X5).addReg(RISCV::X7).addImm(
7822 FunctionAddressOffset)),
7826 MCInstBuilder(RISCV::LD).addReg(RISCV::X7).addReg(RISCV::X7).addImm(
7827 StaticChainOffset)),
7846 SDValue FunctionAddress =
Op.getOperand(2);
7850 struct OffsetValuePair {
7854 } OffsetValues[] = {
7855 {StaticChainOffset, StaticChain},
7856 {FunctionAddressOffset, FunctionAddress},
7861 DAG.
getConstant(OffsetValue.Offset, dl, MVT::i64));
7862 OffsetValue.Addr =
Addr;
7863 OutChains[
Idx + 4] =
7872 SDValue EndOfTrmp = OffsetValues[0].Addr;
7886 return Op.getOperand(0);
7903 N->getOffset(), Flags);
7932template <
class NodeTy>
7934 bool IsLocal,
bool IsExternWeak)
const {
7944 if (IsLocal && !Subtarget.allowTaggedGlobals())
8014 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
8023 return getAddr(
N, DAG);
8030 return getAddr(
N, DAG);
8037 return getAddr(
N, DAG);
8042 bool UseGOT)
const {
8106 Args.push_back(Entry);
8139 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
8153 Addr = getStaticTLSAddr(
N, DAG,
false);
8156 Addr = getStaticTLSAddr(
N, DAG,
true);
8161 : getDynamicTLSAddr(
N, DAG);
8178 if (
LHS == LHS2 &&
RHS == RHS2) {
8183 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
8191 return std::nullopt;
8199 MVT VT =
N->getSimpleValueType(0);
8229 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
8232 if (~TrueVal == FalseVal) {
8272 if (Subtarget.hasShortForwardBranchOpt())
8275 unsigned SelOpNo = 0;
8285 unsigned ConstSelOpNo = 1;
8286 unsigned OtherSelOpNo = 2;
8287 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
8292 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
8293 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
8297 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
8298 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
8304 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
8306 std::swap(NewConstOps[0], NewConstOps[1]);
8318 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
8320 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
8323 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
8324 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
8333 MVT VT =
Op.getSimpleValueType();
8347 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
8375 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
8379 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
8381 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
8382 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
8384 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
8389 DL, VT, LHSVal, CondV);
8405 if (
Op.hasOneUse()) {
8406 unsigned UseOpc =
Op->user_begin()->getOpcode();
8415 return lowerSELECT(NewSel, DAG);
8444 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
8465 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
8469 if (TrueVal - 1 == FalseVal)
8471 if (TrueVal + 1 == FalseVal)
8478 RHS == TrueV && LHS == FalseV) {
8495 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
8521 LHS, RHS, TargetCC,
Op.getOperand(2));
8539 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
8551 int XLenInBytes = Subtarget.
getXLen() / 8;
8553 EVT VT =
Op.getValueType();
8556 unsigned Depth =
Op.getConstantOperandVal(0);
8558 int Offset = -(XLenInBytes * 2);
8575 int XLenInBytes = Subtarget.
getXLen() / 8;
8580 EVT VT =
Op.getValueType();
8582 unsigned Depth =
Op.getConstantOperandVal(0);
8584 int Off = -XLenInBytes;
8585 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
8604 EVT VT =
Lo.getValueType();
8643 EVT VT =
Lo.getValueType();
8694 MVT VT =
Op.getSimpleValueType();
8720 MVT VecVT =
Op.getSimpleValueType();
8722 "Unexpected SPLAT_VECTOR_PARTS lowering");
8728 MVT ContainerVT = VecVT;
8748 int64_t ExtTrueVal)
const {
8750 MVT VecVT =
Op.getSimpleValueType();
8753 assert(Src.getValueType().isVector() &&
8754 Src.getValueType().getVectorElementType() == MVT::i1);
8775 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8777 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
8780 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
8785SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
8787 MVT ExtVT =
Op.getSimpleValueType();
8791 MVT VT =
Op.getOperand(0).getSimpleValueType();
8817 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8819 EVT MaskVT =
Op.getValueType();
8822 "Unexpected type for vector mask lowering");
8824 MVT VecVT = Src.getSimpleValueType();
8828 VL =
Op.getOperand(2);
8831 MVT ContainerVT = VecVT;
8837 MVT MaskContainerVT =
8844 std::tie(Mask, VL) =
8852 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8854 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8858 DAG.
getUNDEF(ContainerVT), Mask, VL);
8861 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8869 unsigned Opc =
Op.getOpcode();
8870 bool IsVPTrunc = Opc == ISD::VP_TRUNCATE;
8873 MVT VT =
Op.getSimpleValueType();
8875 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8879 return lowerVectorMaskTruncLike(
Op, DAG);
8887 MVT SrcVT = Src.getSimpleValueType();
8892 "Unexpected vector truncate lowering");
8894 MVT ContainerVT = SrcVT;
8898 VL =
Op.getOperand(2);
8911 std::tie(Mask, VL) =
8927 }
while (SrcEltVT != DstEltVT);
8936RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8941 MVT VT =
Op.getSimpleValueType();
8942 MVT SrcVT = Src.getSimpleValueType();
8943 MVT ContainerVT = VT;
8966 Chain, Src, Mask, VL);
8967 Chain = Src.getValue(1);
8974 Chain, Src, Mask, VL);
8985RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8988 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8995 MVT VT =
Op.getSimpleValueType();
8997 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
9000 MVT SrcVT = Src.getSimpleValueType();
9002 bool IsDirectExtend =
9010 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
9013 MVT ContainerVT = VT;
9017 VL =
Op.getOperand(2);
9031 std::tie(Mask, VL) =
9037 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
9043 unsigned InterConvOpc =
9048 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
9050 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
9061static std::optional<MVT>
9067 const unsigned MinVLMAX = VectorBitsMin / EltSize;
9069 if (MaxIdx < MinVLMAX)
9071 else if (MaxIdx < MinVLMAX * 2)
9073 else if (MaxIdx < MinVLMAX * 4)
9078 return std::nullopt;
9091 MVT VecVT =
Op.getSimpleValueType();
9108 ValVT == MVT::bf16) {
9117 MVT ContainerVT = VecVT;
9126 MVT OrigContainerVT = ContainerVT;
9129 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
9130 const unsigned OrigIdx = IdxC->getZExtValue();
9133 DL, DAG, Subtarget)) {
9134 ContainerVT = *ShrunkVT;
9143 VLEN && ContainerVT.
bitsGT(M1VT)) {
9146 unsigned RemIdx = OrigIdx % ElemsPerVReg;
9147 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
9148 unsigned ExtractIdx =
9165 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
9166 const auto *CVal = cast<ConstantSDNode>(Val);
9167 if (isInt<32>(CVal->getSExtValue())) {
9168 IsLegalInsert =
true;
9177 if (IsLegalInsert) {
9183 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
9199 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
9200 MVT I32ContainerVT =
9211 Vec, Vec, ValLo, I32Mask, InsertI64VL);
9216 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
9218 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
9223 ValInVec, AlignedIdx);
9233 DAG.
getUNDEF(I32ContainerVT), ValLo,
9234 I32Mask, InsertI64VL);
9236 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
9237 I32Mask, InsertI64VL);
9239 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
9252 Idx, Mask, InsertVL, Policy);
9256 Slideup, AlignedIdx);
9271 EVT EltVT =
Op.getValueType();
9278 MVT ContainerVT = VecVT;
9294 unsigned WidenVecLen;
9297 unsigned MaxEEW = Subtarget.
getELen();
9302 "the number of elements should be power of 2");
9306 ExtractBitIdx =
Idx;
9308 WideEltVT = LargestEltVT;
9311 ExtractElementIdx = DAG.
getNode(
9322 Vec, ExtractElementIdx);
9338 EltVT == MVT::bf16) {
9348 MVT ContainerVT = VecVT;
9359 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
9360 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
9362 unsigned OrigIdx = IdxC->getZExtValue();
9365 unsigned RemIdx = OrigIdx % ElemsPerVReg;
9366 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
9367 unsigned ExtractIdx =
9377 std::optional<uint64_t> MaxIdx;
9380 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
9381 MaxIdx = IdxC->getZExtValue();
9383 if (
auto SmallerVT =
9385 ContainerVT = *SmallerVT;
9432 "Unexpected opcode");
9439 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9444 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9445 if (!
II || !
II->hasScalarOperand())
9448 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
9461 if (OpVT.
bitsLT(XLenVT)) {
9468 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9478 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
9479 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
9482 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
9493 case Intrinsic::riscv_vslide1up:
9494 case Intrinsic::riscv_vslide1down:
9495 case Intrinsic::riscv_vslide1up_mask:
9496 case Intrinsic::riscv_vslide1down_mask: {
9499 bool IsMasked = NumOps == 7;
9505 std::tie(ScalarLo, ScalarHi) =
9513 if (isa<ConstantSDNode>(AVL)) {
9514 const auto [MinVLMAX, MaxVLMAX] =
9518 if (AVLInt <= MinVLMAX) {
9520 }
else if (AVLInt >= 2 * MaxVLMAX) {
9554 if (IntNo == Intrinsic::riscv_vslide1up ||
9555 IntNo == Intrinsic::riscv_vslide1up_mask) {
9557 ScalarHi, I32Mask, I32VL);
9559 ScalarLo, I32Mask, I32VL);
9562 ScalarLo, I32Mask, I32VL);
9564 ScalarHi, I32Mask, I32VL);
9613 const unsigned ElementWidth = 8;
9618 [[maybe_unused]]
unsigned MinVF =
9621 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
9625 bool Fractional = VF < LMul1VF;
9626 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
9647 MVT ContainerVT = OpVT;
9674 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9678 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9679 if (!
II || !
II->hasScalarOperand())
9682 unsigned SplatOp =
II->ScalarOperand + 1;
9695 if (OpVT.
bitsLT(XLenVT)) {
9698 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9709 EVT ValType = V.getValueType();
9710 if (ValType.isVector() && ValType.isFloatingPoint()) {
9713 ValType.getVectorElementCount());
9716 if (ValType.isFixedLengthVector()) {
9718 DAG, V.getSimpleValueType(), Subtarget);
9734 unsigned IntNo =
Op.getConstantOperandVal(0);
9741 case Intrinsic::riscv_tuple_insert: {
9749 case Intrinsic::riscv_tuple_extract: {
9756 case Intrinsic::thread_pointer: {
9760 case Intrinsic::riscv_orc_b:
9761 case Intrinsic::riscv_brev8:
9762 case Intrinsic::riscv_sha256sig0:
9763 case Intrinsic::riscv_sha256sig1:
9764 case Intrinsic::riscv_sha256sum0:
9765 case Intrinsic::riscv_sha256sum1:
9766 case Intrinsic::riscv_sm3p0:
9767 case Intrinsic::riscv_sm3p1: {
9780 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9782 case Intrinsic::riscv_sm4ks:
9783 case Intrinsic::riscv_sm4ed: {
9787 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
9790 case Intrinsic::riscv_zip:
9791 case Intrinsic::riscv_unzip: {
9794 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9796 case Intrinsic::riscv_mopr:
9800 case Intrinsic::riscv_moprr: {
9802 Op.getOperand(2),
Op.getOperand(3));
9804 case Intrinsic::riscv_clmul:
9807 case Intrinsic::riscv_clmulh:
9808 case Intrinsic::riscv_clmulr: {
9811 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9813 case Intrinsic::experimental_get_vector_length:
9815 case Intrinsic::experimental_cttz_elts:
9817 case Intrinsic::riscv_vmv_x_s: {
9821 case Intrinsic::riscv_vfmv_f_s:
9824 case Intrinsic::riscv_vmv_v_x:
9826 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9828 case Intrinsic::riscv_vfmv_v_f:
9830 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9831 case Intrinsic::riscv_vmv_s_x: {
9834 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9837 Op.getOperand(1), Scalar,
Op.getOperand(3));
9840 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9857 MVT VT =
Op.getSimpleValueType();
9862 if (
Op.getOperand(1).isUndef())
9878 case Intrinsic::riscv_vfmv_s_f:
9880 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9882 case Intrinsic::riscv_vaesdf_vv:
9883 case Intrinsic::riscv_vaesdf_vs:
9884 case Intrinsic::riscv_vaesdm_vv:
9885 case Intrinsic::riscv_vaesdm_vs:
9886 case Intrinsic::riscv_vaesef_vv:
9887 case Intrinsic::riscv_vaesef_vs:
9888 case Intrinsic::riscv_vaesem_vv:
9889 case Intrinsic::riscv_vaesem_vs:
9890 case Intrinsic::riscv_vaeskf1:
9891 case Intrinsic::riscv_vaeskf2:
9892 case Intrinsic::riscv_vaesz_vs:
9893 case Intrinsic::riscv_vsm4k:
9894 case Intrinsic::riscv_vsm4r_vv:
9895 case Intrinsic::riscv_vsm4r_vs: {
9896 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9897 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9898 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9903 case Intrinsic::riscv_vsm3c:
9904 case Intrinsic::riscv_vsm3me: {
9905 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9906 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9911 case Intrinsic::riscv_vsha2ch:
9912 case Intrinsic::riscv_vsha2cl:
9913 case Intrinsic::riscv_vsha2ms: {
9914 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9915 !Subtarget.hasStdExtZvknhb())
9917 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9918 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9919 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9923 case Intrinsic::riscv_sf_vc_v_x:
9924 case Intrinsic::riscv_sf_vc_v_i:
9925 case Intrinsic::riscv_sf_vc_v_xv:
9926 case Intrinsic::riscv_sf_vc_v_iv:
9927 case Intrinsic::riscv_sf_vc_v_vv:
9928 case Intrinsic::riscv_sf_vc_v_fv:
9929 case Intrinsic::riscv_sf_vc_v_xvv:
9930 case Intrinsic::riscv_sf_vc_v_ivv:
9931 case Intrinsic::riscv_sf_vc_v_vvv:
9932 case Intrinsic::riscv_sf_vc_v_fvv:
9933 case Intrinsic::riscv_sf_vc_v_xvw:
9934 case Intrinsic::riscv_sf_vc_v_ivw:
9935 case Intrinsic::riscv_sf_vc_v_vvw:
9936 case Intrinsic::riscv_sf_vc_v_fvw: {
9937 MVT VT =
Op.getSimpleValueType();
9974 MVT VT =
Op.getSimpleValueType();
9978 if (VT.isFloatingPoint()) {
9983 if (VT.isFixedLengthVector())
9993 if (VT.isFixedLengthVector())
9995 if (VT.isFloatingPoint())
10014 unsigned IntNo =
Op.getConstantOperandVal(1);
10018 case Intrinsic::riscv_seg2_load:
10019 case Intrinsic::riscv_seg3_load:
10020 case Intrinsic::riscv_seg4_load:
10021 case Intrinsic::riscv_seg5_load:
10022 case Intrinsic::riscv_seg6_load:
10023 case Intrinsic::riscv_seg7_load:
10024 case Intrinsic::riscv_seg8_load: {
10027 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
10028 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
10029 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
10030 Intrinsic::riscv_vlseg8};
10031 unsigned NF =
Op->getNumValues() - 1;
10032 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
10034 MVT VT =
Op->getSimpleValueType(0);
10042 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
10054 Load->getMemoryVT(),
Load->getMemOperand());
10056 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
10065 case Intrinsic::riscv_sf_vc_v_x_se:
10067 case Intrinsic::riscv_sf_vc_v_i_se:
10069 case Intrinsic::riscv_sf_vc_v_xv_se:
10071 case Intrinsic::riscv_sf_vc_v_iv_se:
10073 case Intrinsic::riscv_sf_vc_v_vv_se:
10075 case Intrinsic::riscv_sf_vc_v_fv_se:
10077 case Intrinsic::riscv_sf_vc_v_xvv_se:
10079 case Intrinsic::riscv_sf_vc_v_ivv_se:
10081 case Intrinsic::riscv_sf_vc_v_vvv_se:
10083 case Intrinsic::riscv_sf_vc_v_fvv_se:
10085 case Intrinsic::riscv_sf_vc_v_xvw_se:
10087 case Intrinsic::riscv_sf_vc_v_ivw_se:
10089 case Intrinsic::riscv_sf_vc_v_vvw_se:
10091 case Intrinsic::riscv_sf_vc_v_fvw_se:
10100 unsigned IntNo =
Op.getConstantOperandVal(1);
10104 case Intrinsic::riscv_seg2_store:
10105 case Intrinsic::riscv_seg3_store:
10106 case Intrinsic::riscv_seg4_store:
10107 case Intrinsic::riscv_seg5_store:
10108 case Intrinsic::riscv_seg6_store:
10109 case Intrinsic::riscv_seg7_store:
10110 case Intrinsic::riscv_seg8_store: {
10113 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
10114 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
10115 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
10116 Intrinsic::riscv_vsseg8};
10119 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
10121 MVT VT =
Op->getOperand(2).getSimpleValueType();
10131 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
10134 for (
unsigned i = 0; i < NF; i++)
10138 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget),
10142 FixedIntrinsic->getChain(),
10151 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
10153 case Intrinsic::riscv_sf_vc_xv_se:
10155 case Intrinsic::riscv_sf_vc_iv_se:
10157 case Intrinsic::riscv_sf_vc_vv_se:
10159 case Intrinsic::riscv_sf_vc_fv_se:
10161 case Intrinsic::riscv_sf_vc_xvv_se:
10163 case Intrinsic::riscv_sf_vc_ivv_se:
10165 case Intrinsic::riscv_sf_vc_vvv_se:
10167 case Intrinsic::riscv_sf_vc_fvv_se:
10169 case Intrinsic::riscv_sf_vc_xvw_se:
10171 case Intrinsic::riscv_sf_vc_ivw_se:
10173 case Intrinsic::riscv_sf_vc_vvw_se:
10175 case Intrinsic::riscv_sf_vc_fvw_se:
10183 switch (ISDOpcode) {
10186 case ISD::VP_REDUCE_ADD:
10189 case ISD::VP_REDUCE_UMAX:
10192 case ISD::VP_REDUCE_SMAX:
10195 case ISD::VP_REDUCE_UMIN:
10198 case ISD::VP_REDUCE_SMIN:
10201 case ISD::VP_REDUCE_AND:
10204 case ISD::VP_REDUCE_OR:
10207 case ISD::VP_REDUCE_XOR:
10210 case ISD::VP_REDUCE_FADD:
10212 case ISD::VP_REDUCE_SEQ_FADD:
10214 case ISD::VP_REDUCE_FMAX:
10215 case ISD::VP_REDUCE_FMAXIMUM:
10217 case ISD::VP_REDUCE_FMIN:
10218 case ISD::VP_REDUCE_FMINIMUM:
10228 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
10233 Op.getOpcode() == ISD::VP_REDUCE_AND ||
10234 Op.getOpcode() == ISD::VP_REDUCE_OR ||
10235 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
10236 "Unexpected reduction lowering");
10240 MVT ContainerVT = VecVT;
10248 Mask =
Op.getOperand(2);
10249 VL =
Op.getOperand(3);
10251 std::tie(Mask, VL) =
10256 switch (
Op.getOpcode()) {
10260 case ISD::VP_REDUCE_AND: {
10272 case ISD::VP_REDUCE_OR:
10278 case ISD::VP_REDUCE_XOR: {
10302 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
10306 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
10307 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
10308 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
10309 (ImmAVL && ImmAVL->getZExtValue() >= 1);
10325 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
10329 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
10332 if (M1VT != InnerVT)
10338 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
10357 VecEVT =
Lo.getValueType();
10370 MVT ContainerVT = VecVT;
10390 Mask, VL,
DL, DAG, Subtarget);
10396static std::tuple<unsigned, SDValue, SDValue>
10400 auto Flags =
Op->getFlags();
10401 unsigned Opcode =
Op.getOpcode();
10425 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
10433 MVT VecEltVT =
Op.getSimpleValueType();
10435 unsigned RVVOpcode;
10436 SDValue VectorVal, ScalarVal;
10437 std::tie(RVVOpcode, VectorVal, ScalarVal) =
10441 MVT ContainerVT = VecVT;
10447 MVT ResVT =
Op.getSimpleValueType();
10450 VL,
DL, DAG, Subtarget);
10455 if (
Op->getFlags().hasNoNaNs())
10461 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
10462 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
10468 DL, ResVT, NoNaNs, Res,
10475 unsigned Opc =
Op.getOpcode();
10498 Vec, Mask, VL,
DL, DAG, Subtarget);
10499 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
10500 Op->getFlags().hasNoNaNs())
10517 DL, ResVT, NoNaNs, Res,
10530 unsigned OrigIdx =
Op.getConstantOperandVal(2);
10533 if (OrigIdx == 0 && Vec.
isUndef())
10544 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10547 "Unexpected mask vector lowering");
10579 MVT ContainerVT = VecVT;
10586 DAG.
getUNDEF(ContainerVT), SubVec,
10603 if (OrigIdx == 0) {
10608 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
10609 SlideupAmt, Mask, VL, Policy);
10617 MVT ContainerVecVT = VecVT;
10623 MVT ContainerSubVecVT = SubVecVT;
10629 unsigned SubRegIdx;
10639 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10640 SubRegIdx = Decompose.first;
10642 (OrigIdx % Vscale));
10646 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
10647 SubRegIdx = Decompose.first;
10654 bool ExactlyVecRegSized =
10656 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
10671 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10675 if (SubRegIdx == RISCV::NoSubRegister) {
10698 MVT InterSubVT = ContainerVecVT;
10699 SDValue AlignedExtract = Vec;
10739 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10740 SlideupAmt, Mask, VL, Policy);
10745 if (ContainerVecVT.
bitsGT(InterSubVT))
10754 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10760 MVT SubVecVT =
Op.getSimpleValueType();
10765 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10781 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10784 "Unexpected mask vector lowering");
10818 MVT ContainerVT = VecVT;
10826 if (
auto ShrunkVT =
10828 ContainerVT = *ShrunkVT;
10841 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10853 MVT ContainerSubVecVT = SubVecVT;
10857 unsigned SubRegIdx;
10867 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10868 SubRegIdx = Decompose.first;
10870 (OrigIdx % Vscale));
10874 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10875 SubRegIdx = Decompose.first;
10902 MVT InterSubVT = VecVT;
10907 assert(SubRegIdx != RISCV::NoSubRegister);
10927 Vec, SlidedownAmt, Mask, VL);
10936 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10943 MVT VT =
N.getSimpleValueType();
10947 assert(
Op.getSimpleValueType() == VT &&
10948 "Operands and result must be same type");
10952 unsigned NumVals =
N->getNumValues();
10955 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10958 for (
unsigned I = 0;
I < NumVals;
I++) {
10964 if (TruncVals.
size() > 1)
10966 return TruncVals.
front();
10972 MVT VecVT =
Op.getSimpleValueType();
10975 "vector_interleave on non-scalable vector!");
10986 EVT SplitVT = Op0Lo.getValueType();
10989 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10991 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
11005 Op.getOperand(0),
Op.getOperand(1));
11024 EvenSplat = DAG.
getBitcast(MVT::nxv64i1, EvenSplat);
11029 OddSplat = DAG.
getBitcast(MVT::nxv64i1, OddSplat);
11035 EvenMask, DAG.
getUNDEF(ConcatVT));
11051 MVT VecVT =
Op.getSimpleValueType();
11054 "vector_interleave on non-scalable vector!");
11067 EVT SplitVT = Op0Lo.getValueType();
11070 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
11072 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
11094 Op.getOperand(0),
Op.getOperand(1));
11142 MVT VT =
Op.getSimpleValueType();
11147 uint64_t StepValImm =
Op.getConstantOperandVal(0);
11148 if (StepValImm != 1) {
11157 VL, VT,
DL, DAG, Subtarget);
11172 MVT VecVT =
Op.getSimpleValueType();
11181 MVT ContainerVT = VecVT;
11235 unsigned MaxVLMAX =
11245 if (MaxVLMAX > 256 && EltSize == 8) {
11274 assert(isUInt<16>(MaxVLMAX - 1));
11300 DAG.
getUNDEF(ContainerVT), Mask, VL);
11312 MVT VecVT =
Op.getSimpleValueType();
11316 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
11317 SDValue DownOffset, UpOffset;
11318 if (ImmValue >= 0) {
11334 DownOffset, TrueMask, UpOffset);
11335 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
11341RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
11344 auto *
Load = cast<LoadSDNode>(
Op);
11347 Load->getMemoryVT(),
11348 *
Load->getMemOperand()) &&
11349 "Expecting a correctly-aligned load");
11351 MVT VT =
Op.getSimpleValueType();
11357 const auto [MinVLMAX, MaxVLMAX] =
11360 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
11374 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
11377 Ops.push_back(DAG.
getUNDEF(ContainerVT));
11378 Ops.push_back(
Load->getBasePtr());
11383 Load->getMemoryVT(),
Load->getMemOperand());
11390RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
11393 auto *
Store = cast<StoreSDNode>(
Op);
11396 Store->getMemoryVT(),
11397 *
Store->getMemOperand()) &&
11398 "Expecting a correctly-aligned store");
11419 const auto [MinVLMAX, MaxVLMAX] =
11422 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
11433 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
11436 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
11437 Store->getMemoryVT(),
Store->getMemOperand());
11443 MVT VT =
Op.getSimpleValueType();
11445 const auto *MemSD = cast<MemSDNode>(
Op);
11446 EVT MemVT = MemSD->getMemoryVT();
11448 SDValue Chain = MemSD->getChain();
11452 bool IsExpandingLoad =
false;
11453 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
11454 Mask = VPLoad->getMask();
11456 VL = VPLoad->getVectorLength();
11458 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
11459 Mask = MLoad->getMask();
11460 PassThru = MLoad->getPassThru();
11461 IsExpandingLoad = MLoad->isExpandingLoad();
11468 MVT ContainerVT = VT;
11482 if (!IsUnmasked && IsExpandingLoad) {
11489 unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
11490 : Intrinsic::riscv_vle_mask;
11492 if (IntID == Intrinsic::riscv_vle)
11493 Ops.push_back(DAG.
getUNDEF(ContainerVT));
11495 Ops.push_back(PassThru);
11496 Ops.push_back(BasePtr);
11497 if (IntID == Intrinsic::riscv_vle_mask)
11498 Ops.push_back(Mask);
11500 if (IntID == Intrinsic::riscv_vle_mask)
11507 Chain =
Result.getValue(1);
11509 MVT IndexVT = ContainerVT;
11514 bool UseVRGATHEREI16 =
false;
11522 UseVRGATHEREI16 =
true;
11528 DAG.
getUNDEF(IndexVT), Mask, ExpandingVL);
11532 DL, ContainerVT, Result, Iota, PassThru, Mask, ExpandingVL);
11545 const auto *MemSD = cast<MemSDNode>(
Op);
11546 EVT MemVT = MemSD->getMemoryVT();
11548 SDValue Chain = MemSD->getChain();
11552 bool IsCompressingStore =
false;
11553 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
11554 Val = VPStore->getValue();
11555 Mask = VPStore->getMask();
11556 VL = VPStore->getVectorLength();
11558 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
11559 Val = MStore->getValue();
11560 Mask = MStore->getMask();
11561 IsCompressingStore = MStore->isCompressingStore();
11570 MVT ContainerVT = VT;
11575 if (!IsUnmasked || IsCompressingStore) {
11584 if (IsCompressingStore) {
11587 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
11594 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
11596 Ops.push_back(Val);
11597 Ops.push_back(BasePtr);
11599 Ops.push_back(Mask);
11603 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11615 MVT ContainerVT = VT;
11628 Passthru, Val, Mask, VL);
11637RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
11639 MVT InVT =
Op.getOperand(0).getSimpleValueType();
11642 MVT VT =
Op.getSimpleValueType();
11656 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
11663 unsigned Opc =
Op.getOpcode();
11670 MVT VT =
Op.getSimpleValueType();
11703 MVT ContainerInVT = InVT;
11722 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11726 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11734 {Chain, Op1, Op2, CC, Mask, Mask, VL});
11739 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
11752 MVT VT =
Op.getSimpleValueType();
11756 "Unexpected type for ISD::ABS");
11758 MVT ContainerVT = VT;
11765 if (
Op->getOpcode() == ISD::VP_ABS) {
11766 Mask =
Op->getOperand(1);
11770 VL =
Op->getOperand(2);
11778 DAG.
getUNDEF(ContainerVT), Mask, VL);
11780 DAG.
getUNDEF(ContainerVT), Mask, VL);
11787SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
11790 MVT VT =
Op.getSimpleValueType();
11794 "Can only handle COPYSIGN with matching types.");
11803 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
11808SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
11810 MVT VT =
Op.getSimpleValueType();
11813 MVT I1ContainerVT =
11827 Op2, DAG.
getUNDEF(ContainerVT), VL);
11838 MVT VT =
Op.getSimpleValueType();
11843 for (
const SDValue &V :
Op->op_values()) {
11844 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11847 if (!
V.getValueType().isVector()) {
11853 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
11854 "Only fixed length vectors are supported!");
11868 if (
Op->isStrictFPOpcode()) {
11877 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11891 MVT VT =
Op.getSimpleValueType();
11894 MVT ContainerVT = VT;
11900 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11903 if (HasPassthruOp) {
11906 if (*MaskIdx == OpIdx.index())
11910 if (
Op.getOpcode() == ISD::VP_MERGE) {
11914 assert(
Op.getOpcode() == ISD::VP_SELECT);
11926 if (!
V.getValueType().isFixedLengthVector()) {
11931 MVT OpVT =
V.getSimpleValueType();
11933 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11934 "Only fixed length vectors are supported!");
11939 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11949 MVT VT =
Op.getSimpleValueType();
11955 MVT ContainerVT = VT;
11965 DAG.
getUNDEF(ContainerVT), Zero, VL);
11968 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11970 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11973 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11982 MVT VT =
Op.getSimpleValueType();
11986 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11990 MVT ContainerVT = VT;
12000 switch (Condition) {
12068 MVT DstVT =
Op.getSimpleValueType();
12069 MVT SrcVT = Src.getSimpleValueType();
12082 if (DstEltSize >= SrcEltSize) {
12091 if (SrcEltSize == 1) {
12102 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
12103 }
else if (DstEltSize > (2 * SrcEltSize)) {
12107 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
12113 "Wrong input/output vector types");
12116 if (DstEltSize > (2 * SrcEltSize)) {
12132 MVT InterimFVT = DstVT;
12133 if (SrcEltSize > (2 * DstEltSize)) {
12134 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
12141 if (InterimFVT != DstVT) {
12147 "Wrong input/output vector types");
12151 if (DstEltSize == 1) {
12154 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
12164 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
12174 while (InterimIVT != DstVT) {
12186 MVT VT =
Op.getSimpleValueType();
12195 MVT VT =
Op.getSimpleValueType();
12209 MVT ContainerVT = VT;
12230 SplatZero, DAG.
getUNDEF(PromotedVT), VL);
12233 SplatOne, SplatZero, DAG.
getUNDEF(PromotedVT), VLMax);
12237 TrueVal, FalseVal, FalseVal, VL);
12252RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
12264 MVT VT =
Op.getSimpleValueType();
12265 MVT ContainerVT = VT;
12275 if (IsMaskVector) {
12286 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
12295 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
12298 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
12299 SDValue DownOffset, UpOffset;
12300 if (ImmValue >= 0) {
12314 Op1, DownOffset, Mask, UpOffset);
12318 if (IsMaskVector) {
12322 {Result, DAG.getConstant(0, DL, ContainerVT),
12323 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
12338 MVT VT =
Op.getSimpleValueType();
12340 MVT ContainerVT = VT;
12356RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
12359 MVT VT =
Op.getSimpleValueType();
12366 MVT ContainerVT = VT;
12374 MVT GatherVT = ContainerVT;
12378 if (IsMaskVector) {
12389 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
12395 unsigned MaxVLMAX =
12404 if (MaxVLMAX > 256 && EltSize == 8) {
12432 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
12434 if (IsMaskVector) {
12457 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
12459 DAG.
getUNDEF(IndicesVT), Mask, EVL);
12461 DAG.
getUNDEF(GatherVT), Mask, EVL);
12463 if (IsMaskVector) {
12478 MVT VT =
Op.getSimpleValueType();
12480 return lowerVPOp(
Op, DAG);
12487 MVT ContainerVT = VT;
12506 MVT VT =
Op.getSimpleValueType();
12507 MVT ContainerVT = VT;
12513 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
12519 : Intrinsic::riscv_vlse_mask,
12522 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
12523 VPNode->getStride()};
12531 Ops.
push_back(VPNode->getVectorLength());
12539 VPNode->getMemoryVT(), VPNode->getMemOperand());
12553 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
12554 SDValue StoreVal = VPNode->getValue();
12556 MVT ContainerVT = VT;
12567 : Intrinsic::riscv_vsse_mask,
12570 VPNode->getBasePtr(), VPNode->getStride()};
12578 Ops.
push_back(VPNode->getVectorLength());
12581 Ops, VPNode->getMemoryVT(),
12582 VPNode->getMemOperand());
12594 MVT VT =
Op.getSimpleValueType();
12596 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
12597 EVT MemVT = MemSD->getMemoryVT();
12599 SDValue Chain = MemSD->getChain();
12605 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
12606 Index = VPGN->getIndex();
12607 Mask = VPGN->getMask();
12609 VL = VPGN->getVectorLength();
12614 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
12615 Index = MGN->getIndex();
12616 Mask = MGN->getMask();
12617 PassThru = MGN->getPassThru();
12621 MVT IndexVT =
Index.getSimpleValueType();
12625 "Unexpected VTs!");
12626 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12629 "Unexpected extending MGATHER/VP_GATHER");
12635 MVT ContainerVT = VT;
12659 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
12676 Chain =
Result.getValue(1);
12693 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
12694 EVT MemVT = MemSD->getMemoryVT();
12696 SDValue Chain = MemSD->getChain();
12699 [[maybe_unused]]
bool IsTruncatingStore =
false;
12702 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
12703 Index = VPSN->getIndex();
12704 Mask = VPSN->getMask();
12705 Val = VPSN->getValue();
12706 VL = VPSN->getVectorLength();
12708 IsTruncatingStore =
false;
12711 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
12712 Index = MSN->getIndex();
12713 Mask = MSN->getMask();
12714 Val = MSN->getValue();
12715 IsTruncatingStore = MSN->isTruncatingStore();
12719 MVT IndexVT =
Index.getSimpleValueType();
12723 "Unexpected VTs!");
12724 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12727 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
12733 MVT ContainerVT = VT;
12757 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
12767 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
12783 static const int Table =
12812 static const unsigned Table =
12835 bool isRISCV64 = Subtarget.
is64Bit();
12899 switch (
N->getOpcode()) {
12901 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12907 "Unexpected custom legalisation");
12908 bool IsStrict =
N->isStrictFPOpcode();
12911 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12929 Opc,
DL, VTs, Chain, Op0,
12963 std::tie(Result, Chain) =
12964 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12992 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
13003 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
13004 "has custom type legalization on riscv32");
13006 SDValue LoCounter, HiCounter;
13017 N->getOperand(0), LoCounter, HiCounter);
13041 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
13042 unsigned XLen = Subtarget.
getXLen();
13045 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
13053 if (LHSIsU == RHSIsU)
13070 if (RHSIsU && LHSIsS && !RHSIsS)
13072 else if (LHSIsU && RHSIsS && !LHSIsS)
13082 "Unexpected custom legalisation");
13089 "Unexpected custom legalisation");
13092 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
13118 "Unexpected custom legalisation");
13119 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
13120 Subtarget.hasVendorXTHeadBb()) &&
13121 "Unexpected custom legalization");
13122 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
13123 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
13132 "Unexpected custom legalisation");
13146 MVT VT =
N->getSimpleValueType(0);
13147 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
13148 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
13149 "Unexpected custom legalisation");
13161 if (VT != MVT::i32)
13170 "Unexpected custom legalisation");
13174 if (!isa<ConstantSDNode>(
N->getOperand(1)))
13191 EVT OType =
N->getValueType(1);
13204 "Unexpected custom legalisation");
13221 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
13225 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
13243 !Subtarget.hasStdExtZbb() &&
"Unexpected custom legalisation");
13252 "Unexpected custom legalisation");
13258 "Unexpected custom legalisation");
13260 if (Subtarget.hasStdExtZbb()) {
13294 EVT VT =
N->getValueType(0);
13299 if (VT == MVT::i16 &&
13301 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
13304 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
13309 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.
is64Bit() &&
13312 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
13333 MVT VT =
N->getSimpleValueType(0);
13335 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
13336 "Unexpected custom legalisation");
13339 "Unexpected extension");
13365 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
13367 "Unexpected EXTRACT_VECTOR_ELT legalization");
13370 MVT ContainerVT = VecVT;
13398 DAG.
getUNDEF(ContainerVT), Mask, VL);
13406 unsigned IntNo =
N->getConstantOperandVal(0);
13410 "Don't know how to custom type legalize this intrinsic!");
13411 case Intrinsic::experimental_get_vector_length: {
13416 case Intrinsic::experimental_cttz_elts: {
13422 case Intrinsic::riscv_orc_b:
13423 case Intrinsic::riscv_brev8:
13424 case Intrinsic::riscv_sha256sig0:
13425 case Intrinsic::riscv_sha256sig1:
13426 case Intrinsic::riscv_sha256sum0:
13427 case Intrinsic::riscv_sha256sum1:
13428 case Intrinsic::riscv_sm3p0:
13429 case Intrinsic::riscv_sm3p1: {
13430 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13450 case Intrinsic::riscv_sm4ks:
13451 case Intrinsic::riscv_sm4ed: {
13459 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
13463 case Intrinsic::riscv_mopr: {
13464 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13474 case Intrinsic::riscv_moprr: {
13475 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13487 case Intrinsic::riscv_clmul: {
13488 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13499 case Intrinsic::riscv_clmulh:
13500 case Intrinsic::riscv_clmulr: {
13501 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13529 case Intrinsic::riscv_vmv_x_s: {
13530 EVT VT =
N->getValueType(0);
13532 if (VT.
bitsLT(XLenVT)) {
13541 "Unexpected custom legalization");
13579 case ISD::VP_REDUCE_ADD:
13580 case ISD::VP_REDUCE_AND:
13581 case ISD::VP_REDUCE_OR:
13582 case ISD::VP_REDUCE_XOR:
13583 case ISD::VP_REDUCE_SMAX:
13584 case ISD::VP_REDUCE_UMAX:
13585 case ISD::VP_REDUCE_SMIN:
13586 case ISD::VP_REDUCE_UMIN:
13650 const EVT VT =
N->getValueType(0);
13651 const unsigned Opc =
N->getOpcode();
13658 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
13663 "Inconsistent mappings");
13674 !isa<ConstantSDNode>(
RHS.getOperand(1)))
13677 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
13692 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
13694 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
13695 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
13699 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
13706 if (
LHS.getOpcode() != ReduceOpc)
13721 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
13722 ReduceVec->
getFlags() &
N->getFlags());
13732 auto BinOpToRVVReduce = [](
unsigned Opc) {
13761 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
13764 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
13767 unsigned Opc =
N->getOpcode();
13768 unsigned ReduceIdx;
13769 if (IsReduction(
N->getOperand(0), Opc))
13771 else if (IsReduction(
N->getOperand(1), Opc))
13777 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
13780 SDValue Extract =
N->getOperand(ReduceIdx);
13812 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
13839 if (!Subtarget.hasStdExtZba())
13843 EVT VT =
N->getValueType(0);
13855 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13856 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
13859 int64_t C0 = N0C->getSExtValue();
13860 int64_t C1 = N1C->getSExtValue();
13861 if (C0 <= 0 || C1 <= 0)
13865 int64_t Bits = std::min(C0, C1);
13866 int64_t Diff = std::abs(C0 - C1);
13867 if (Diff != 1 && Diff != 2 && Diff != 3)
13894 EVT VT =
N->getValueType(0);
13902 if ((!Subtarget.hasStdExtZicond() &&
13903 !Subtarget.hasVendorXVentanaCondOps()) ||
13925 bool SwapSelectOps;
13931 SwapSelectOps =
false;
13932 NonConstantVal = FalseVal;
13934 SwapSelectOps =
true;
13935 NonConstantVal = TrueVal;
13941 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13989 EVT VT =
N->getValueType(0);
13997 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13998 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
14004 if (!N0C->hasOneUse())
14006 int64_t C0 = N0C->getSExtValue();
14007 int64_t C1 = N1C->getSExtValue();
14009 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
14012 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
14013 !isInt<12>(C0 * (C1 / C0))) {
14016 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
14017 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
14020 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
14021 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
14048 EVT VT =
N->getValueType(0);
14079 unsigned OuterExtend =
14083 OuterExtend,
SDLoc(
N), VT,
14091 EVT VT =
N->getValueType(0);
14139 EVT VT =
N->getValueType(0);
14143 auto *N0C = dyn_cast<ConstantSDNode>(N0);
14149 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
14159 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
14182 if (!Subtarget.hasStdExtZbb())
14185 EVT VT =
N->getValueType(0);
14187 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
14196 auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
14199 unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
14201 if (ShiftedAmount >= 8)
14205 SDValue RightShiftOperand = N1;
14207 if (ShiftedAmount != 0) {
14210 auto *ShAmtCRight = dyn_cast<ConstantSDNode>(N1.
getOperand(1));
14211 if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
14220 if (LeftShiftOperand != RightShiftOperand)
14224 Mask <<= ShiftedAmount;
14238 EVT VT =
N->getValueType(0);
14269 bool IsAnd =
N->getOpcode() ==
ISD::AND;
14293 EVT VT =
N->getValueType(0);
14317 EVT VT =
N->getValueType(0);
14344 if (CondLHS != True)
14351 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
14363 if (!FalseRHSC || !FalseRHSC->
isZero())
14383 EVT VT =
N->getValueType(0);
14390 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
14417 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
14469 EVT VT =
N->getValueType(0);
14515 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
14540 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
14545 const APInt &Imm = ConstN00->getAPIntValue();
14546 if ((Imm + 1).isSignedIntN(12))
14567 EVT VT =
N->getValueType(0);
14579 const bool HasShlAdd =
14580 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
14594 for (
uint64_t Divisor : {3, 5, 9}) {
14595 if (MulAmt % Divisor != 0)
14597 uint64_t MulAmt2 = MulAmt / Divisor;
14604 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
14605 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
14622 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
14638 if (ScaleShift >= 1 && ScaleShift < 4) {
14639 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
14653 for (
uint64_t Divisor : {3, 5, 9}) {
14658 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
14669 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
14671 if (ScaleShift >= 1 && ScaleShift < 4) {
14672 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
14698 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
14700 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
14711 for (
uint64_t Divisor : {3, 5, 9}) {
14712 if (MulAmt % Divisor != 0)
14714 uint64_t MulAmt2 = MulAmt / Divisor;
14717 for (
uint64_t Divisor2 : {3, 5, 9}) {
14718 if (MulAmt2 % Divisor2 != 0)
14720 uint64_t MulAmt3 = MulAmt2 / Divisor2;
14743 EVT VT =
N->getValueType(0);
14750 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
14751 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
14764 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
14765 V3 != (HalfSize - 1))
14781 EVT VT =
N->getValueType(0);
14789 unsigned AddSubOpc;
14795 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
14796 AddSubOpc = V->getOpcode();
14798 SDValue Opnd = V->getOperand(1);
14799 MulOper = V->getOperand(0);
14808 if (IsAddSubWith1(N0)) {
14810 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
14813 if (IsAddSubWith1(N1)) {
14815 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
14830 if (isIndexTypeSigned(IndexType))
14833 if (!
N->hasOneUse())
14836 EVT VT =
N.getValueType();
14875 EVT SrcVT = Src.getValueType();
14879 NewElen = std::max(NewElen, 8U);
14902 EVT VT =
N->getValueType(0);
14905 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
14909 auto *N1C = dyn_cast<ConstantSDNode>(N1);
14921 if (!isIntEqualitySetCC(
Cond))
14930 const APInt &C1 = N1C->getAPIntValue();
14948 EVT VT =
N->getValueType(0);
14949 EVT SrcVT = cast<VTSDNode>(
N->getOperand(1))->
getVT();
14950 unsigned Opc = Src.getOpcode();
14955 Subtarget.hasStdExtZfhmin())
14957 Src.getOperand(0));
14961 VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
14964 Src.getOperand(1));
14972struct CombineResult;
14974enum ExtKind :
uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
15001struct NodeExtensionHelper {
15010 bool SupportsFPExt;
15013 bool EnforceOneUse;
15028 return OrigOperand;
15039 unsigned getExtOpc(ExtKind SupportsExt)
const {
15040 switch (SupportsExt) {
15041 case ExtKind::SExt:
15043 case ExtKind::ZExt:
15045 case ExtKind::FPExt:
15056 std::optional<ExtKind> SupportsExt)
const {
15057 if (!SupportsExt.has_value())
15058 return OrigOperand;
15060 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
15064 if (
Source.getValueType() == NarrowVT)
15068 if (
Source.getValueType().getVectorElementType() == MVT::bf16) {
15074 unsigned ExtOpc = getExtOpc(*SupportsExt);
15078 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
15085 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
15097 DAG.
getUNDEF(NarrowVT), Source, VL);
15110 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
15116 MVT EltVT = SupportsExt == ExtKind::FPExt
15118 :
MVT::getIntegerVT(NarrowSize);
15120 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
15121 "Trying to extend something we can't represent");
15128 static unsigned getSExtOpcode(
unsigned Opcode) {
15151 static unsigned getZExtOpcode(
unsigned Opcode) {
15177 static unsigned getFPExtOpcode(
unsigned Opcode) {
15202 static unsigned getSUOpcode(
unsigned Opcode) {
15204 "SU is only supported for MUL");
15210 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
15230 using CombineToTry = std::function<std::optional<CombineResult>(
15231 SDNode * ,
const NodeExtensionHelper & ,
15236 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
15240 unsigned Opc = OrigOperand.
getOpcode();
15244 "Unexpected Opcode");
15257 unsigned ScalarBits =
Op.getValueSizeInBits();
15259 if (ScalarBits < EltBits) {
15262 !Subtarget.
is64Bit() &&
"Unexpected splat");
15264 SupportsSExt =
true;
15268 SupportsZExt =
true;
15270 EnforceOneUse =
false;
15274 unsigned NarrowSize = EltBits / 2;
15277 if (NarrowSize < 8)
15281 SupportsSExt =
true;
15285 SupportsZExt =
true;
15287 EnforceOneUse =
false;
15290 bool isSupportedFPExtend(
SDNode *Root,
MVT NarrowEltVT,
15297 if (NarrowEltVT == MVT::bf16 && (!Subtarget.hasStdExtZvfbfwma() ||
15307 SupportsZExt =
false;
15308 SupportsSExt =
false;
15309 SupportsFPExt =
false;
15310 EnforceOneUse =
true;
15311 unsigned Opc = OrigOperand.
getOpcode();
15333 SupportsZExt =
true;
15336 SupportsSExt =
true;
15341 if (!isSupportedFPExtend(Root, NarrowEltVT, Subtarget))
15343 SupportsFPExt =
true;
15348 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
15360 if (!isSupportedFPExtend(Root,
Op.getOperand(0).getSimpleValueType(),
15365 unsigned ScalarBits =
Op.getOperand(0).getValueSizeInBits();
15366 if (NarrowSize != ScalarBits)
15369 SupportsFPExt =
true;
15378 static bool isSupportedRoot(
const SDNode *Root,
15407 Subtarget.hasStdExtZvbb();
15409 return Subtarget.hasStdExtZvbb();
15423 assert(isSupportedRoot(Root, Subtarget) &&
15424 "Trying to build an helper with an "
15425 "unsupported root");
15426 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
15442 if (OperandIdx == 1) {
15451 EnforceOneUse =
false;
15456 fillUpExtensionSupport(Root, DAG, Subtarget);
15462 static std::pair<SDValue, SDValue>
15465 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
15484 switch (
N->getOpcode()) {
15526struct CombineResult {
15528 unsigned TargetOpcode;
15530 std::optional<ExtKind> LHSExt;
15531 std::optional<ExtKind> RHSExt;
15535 NodeExtensionHelper
LHS;
15537 NodeExtensionHelper
RHS;
15539 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
15540 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
15541 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
15542 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
15551 std::tie(Mask, VL) =
15552 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
15566 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
15567 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
15568 Passthru, Mask, VL);
15582static std::optional<CombineResult>
15583canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
15584 const NodeExtensionHelper &RHS,
15587 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
15588 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
15589 Root, LHS, {ExtKind::ZExt}, RHS,
15591 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
15592 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
15593 Root, LHS, {ExtKind::SExt}, RHS,
15595 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
15596 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
15597 Root, LHS, {ExtKind::FPExt}, RHS,
15599 return std::nullopt;
15608static std::optional<CombineResult>
15609canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
15612 return canFoldToVWWithSameExtensionImpl(
15613 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
15621static std::optional<CombineResult>
15622canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
15625 if (
RHS.SupportsFPExt)
15626 return CombineResult(
15627 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
15628 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
15635 return CombineResult(
15636 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
15637 LHS, std::nullopt, RHS, {ExtKind::ZExt});
15639 return CombineResult(
15640 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
15641 LHS, std::nullopt, RHS, {ExtKind::SExt});
15642 return std::nullopt;
15649static std::optional<CombineResult>
15650canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15653 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
15661static std::optional<CombineResult>
15662canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15665 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
15673static std::optional<CombineResult>
15674canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15677 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
15685static std::optional<CombineResult>
15686canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
15690 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
15691 return std::nullopt;
15692 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
15693 Root, LHS, {ExtKind::SExt}, RHS,
15698NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
15709 Strategies.
push_back(canFoldToVWWithSameExtension);
15718 Strategies.
push_back(canFoldToVWWithSameExtension);
15723 Strategies.
push_back(canFoldToVWWithSameExtension);
15730 Strategies.
push_back(canFoldToVWWithZEXT);
15735 Strategies.
push_back(canFoldToVWWithSEXT);
15740 Strategies.
push_back(canFoldToVWWithZEXT);
15745 Strategies.
push_back(canFoldToVWWithFPEXT);
15774 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
15780 Inserted.insert(
N);
15783 while (!Worklist.
empty()) {
15786 NodeExtensionHelper
LHS(Root, 0, DAG, Subtarget);
15787 NodeExtensionHelper
RHS(Root, 1, DAG, Subtarget);
15788 auto AppendUsersIfNeeded = [&Worklist, &Subtarget,
15789 &Inserted](
const NodeExtensionHelper &
Op) {
15790 if (
Op.needToPromoteOtherUsers()) {
15793 if (!NodeExtensionHelper::isSupportedRoot(TheUser, Subtarget))
15798 if (Inserted.insert(TheUser).second)
15811 NodeExtensionHelper::getSupportedFoldings(Root);
15813 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
15814 bool Matched =
false;
15815 for (
int Attempt = 0;
15816 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
15819 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
15820 FoldingStrategies) {
15821 std::optional<CombineResult> Res =
15822 FoldingStrategy(Root,
LHS,
RHS, DAG, Subtarget);
15829 if (Res->LHSExt.has_value())
15830 if (!AppendUsersIfNeeded(
LHS))
15832 if (Res->RHSExt.has_value())
15833 if (!AppendUsersIfNeeded(
RHS))
15845 SDValue InputRootReplacement;
15852 for (CombineResult Res : CombinesToApply) {
15853 SDValue NewValue = Res.materialize(DAG, Subtarget);
15854 if (!InputRootReplacement) {
15856 "First element is expected to be the current node");
15857 InputRootReplacement = NewValue;
15862 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
15866 return InputRootReplacement;
15873 unsigned Opc =
N->getOpcode();
15878 SDValue MergeOp =
N->getOperand(1);
15879 unsigned MergeOpc = MergeOp.
getOpcode();
15890 SDValue Passthru =
N->getOperand(2);
15904 Z = Z.getOperand(1);
15910 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
15917 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
15946 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
15952 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
15954 if (MemVT == MVT::i32)
15960 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
15995 if (!Subtarget.hasVendorXTHeadMemPair())
16007 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
16009 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
16010 return {
Ptr->getOperand(0), C1->getZExtValue()};
16014 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
16035 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
16038 if (Base1 != Base2)
16042 bool Valid =
false;
16043 if (MemVT == MVT::i32) {
16045 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
16047 }
else if (MemVT == MVT::i64) {
16049 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
16083 if (Src->isStrictFPOpcode())
16091 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
16101 EVT VT =
N->getValueType(0);
16104 MVT SrcVT = Src.getSimpleValueType();
16105 MVT SrcContainerVT = SrcVT;
16107 SDValue XVal = Src.getOperand(0);
16134 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
16138 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
16151 if (VT != MVT::i32 && VT != XLenVT)
16181 EVT DstVT =
N->getValueType(0);
16182 if (DstVT != XLenVT)
16188 if (Src->isStrictFPOpcode())
16196 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
16199 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
16208 if (SatVT == DstVT)
16210 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
16216 Src = Src.getOperand(0);
16237 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
16243 EVT VT =
N->getValueType(0);
16258 auto *VPLoad = dyn_cast<VPLoadSDNode>(
N->getOperand(0));
16262 EVT LoadVT = VPLoad->getValueType(0);
16266 N->getOperand(2) != VPLoad->getVectorLength() ||
16267 !
N->getOperand(0).hasOneUse())
16274 SDValue LoadMask = VPLoad->getMask();
16279 if (LoadMask.
getOpcode() != ISD::EXPERIMENTAL_VP_REVERSE ||
16281 LoadMask.
getOperand(2) != VPLoad->getVectorLength())
16289 SDValue NumElem = VPLoad->getVectorLength();
16290 uint64_t ElemWidthByte = VPLoad->getValueType(0).getScalarSizeInBits() / 8;
16302 PtrInfo, VPLoad->getMemOperand()->getFlags(),
16306 LoadVT,
DL, VPLoad->getChain(),
Base, Stride, LoadMask,
16307 VPLoad->getVectorLength(), MMO, VPLoad->isExpandingLoad());
16319 auto *VPStore = cast<VPStoreSDNode>(
N);
16321 if (VPStore->getValue().getOpcode() != ISD::EXPERIMENTAL_VP_REVERSE)
16324 SDValue VPReverse = VPStore->getValue();
16330 VPStore->getVectorLength() != VPReverse.
getOperand(2) ||
16334 SDValue StoreMask = VPStore->getMask();
16339 if (StoreMask.
getOpcode() != ISD::EXPERIMENTAL_VP_REVERSE ||
16341 StoreMask.
getOperand(2) != VPStore->getVectorLength())
16349 SDValue NumElem = VPStore->getVectorLength();
16363 PtrInfo, VPStore->getMemOperand()->getFlags(),
16368 VPStore->getOffset(), Stride, StoreMask, VPStore->getVectorLength(),
16369 VPStore->getMemoryVT(), MMO, VPStore->getAddressingMode(),
16370 VPStore->isTruncatingStore(), VPStore->isCompressingStore());
16419 unsigned Offset = IsStrict ? 1 : 0;
16426 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
16428 V.getOperand(2) == VL) {
16430 V = V.getOperand(0);
16437 bool NegA = invertIfNegative(
A);
16438 bool NegB = invertIfNegative(
B);
16439 bool NegC = invertIfNegative(
C);
16442 if (!NegA && !NegB && !NegC)
16448 {N->getOperand(0), A, B, C, Mask, VL});
16472 EVT VT =
N->getValueType(0);
16477 if (!isa<ConstantSDNode>(
N->getOperand(1)))
16479 uint64_t ShAmt =
N->getConstantOperandVal(1);
16487 cast<VTSDNode>(N0.
getOperand(1))->getVT().getSizeInBits();
16492 if (LShAmt < ExtSize) {
16505 if (ShAmt > 32 || VT != MVT::i64)
16521 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
16534 !isa<ConstantSDNode>(U->getOperand(1)) ||
16535 U->getConstantOperandVal(1) > 32)
16590 if (!
Cond.hasOneUse())
16609 EVT VT =
Cond.getValueType();
16654 LHS =
LHS.getOperand(0);
16664 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
16668 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
16672 RHS =
LHS.getOperand(1);
16673 LHS =
LHS.getOperand(0);
16682 RHS =
LHS.getOperand(1);
16683 LHS =
LHS.getOperand(0);
16699 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
16740 bool Commutative =
true;
16741 unsigned Opc = TrueVal.getOpcode();
16749 Commutative =
false;
16757 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
16761 if (FalseVal == TrueVal.getOperand(0))
16763 else if (Commutative && FalseVal == TrueVal.getOperand(1))
16768 EVT VT =
N->getValueType(0);
16770 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
16776 assert(IdentityOperand &&
"No identity operand!");
16781 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
16782 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
16803 CountZeroes =
N->getOperand(2);
16804 ValOnZero =
N->getOperand(1);
16806 CountZeroes =
N->getOperand(1);
16807 ValOnZero =
N->getOperand(2);
16826 if (
Cond->getOperand(0) != CountZeroesArgument)
16842 CountZeroes, BitWidthMinusOne);
16852 EVT VT =
N->getValueType(0);
16853 EVT CondVT =
Cond.getValueType();
16861 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
16867 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
16888 SDValue TrueVal =
N->getOperand(1);
16889 SDValue FalseVal =
N->getOperand(2);
16904 EVT VT =
N->getValueType(0);
16911 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
16926 if (
Op.isUndef()) {
16939 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
16943 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
16944 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
16948 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
16976 const unsigned InVecOpcode = InVec->
getOpcode();
16986 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
16993 InVecLHS, InValLHS, EltNo);
16995 InVecRHS, InValRHS, EltNo);
17004 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
17007 unsigned Elt = IndexC->getZExtValue();
17015 unsigned ConcatOpIdx = Elt / ConcatNumElts;
17018 ConcatOp, InVal, NewIdx);
17022 ConcatOps[ConcatOpIdx] = ConcatOp;
17034 EVT VT =
N->getValueType(0);
17044 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
17046 !
SDValue(BaseLd, 0).hasOneUse())
17049 EVT BaseLdVT = BaseLd->getValueType(0);
17056 auto *Ld = dyn_cast<LoadSDNode>(
Op);
17057 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
17059 Ld->getValueType(0) != BaseLdVT)
17068 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
17070 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
17075 if (BIO1.equalBaseIndex(BIO2, DAG))
17080 SDValue P2 = Ld2->getBasePtr();
17086 return std::nullopt;
17090 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
17095 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
17096 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
17104 unsigned WideScalarBitWidth =
17117 auto [StrideVariant, MustNegateStride] = *BaseDiff;
17119 std::holds_alternative<SDValue>(StrideVariant)
17120 ? std::get<SDValue>(StrideVariant)
17123 if (MustNegateStride)
17131 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
17132 ConstStride && ConstStride->getSExtValue() >= 0)
17136 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
17142 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
17146 WideVecVT,
DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
17164 EVT VT =
N->getValueType(0);
17192 if (
N->getValueType(0).isFixedLengthVector())
17195 SDValue Addend =
N->getOperand(0);
17199 SDValue AddPassthruOp =
N->getOperand(2);
17200 if (!AddPassthruOp.
isUndef())
17204 auto IsVWMulOpc = [](
unsigned Opc) {
17223 if (!MulPassthruOp.
isUndef())
17233 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
17234 }(
N, DAG, Subtarget);
17239 if (AddMask != MulMask || AddVL != MulVL)
17244 "Unexpected opcode after VWMACC_VL");
17246 "Unexpected opcode after VWMACC_VL!");
17248 "Unexpected opcode after VWMUL_VL!");
17250 "Unexpected opcode after VWMUL_VL!");
17253 EVT VT =
N->getValueType(0);
17269 const EVT IndexVT = Index.getValueType();
17273 if (!isIndexTypeSigned(IndexType))
17305 for (
unsigned i = 0; i < Index->getNumOperands(); i++) {
17308 if (Index->getOperand(i)->isUndef())
17310 uint64_t C = Index->getConstantOperandVal(i);
17311 if (
C % ElementSize != 0)
17313 C =
C / ElementSize;
17317 ActiveLanes.
set(
C);
17319 return ActiveLanes.
all();
17337 if (NumElems % 2 != 0)
17341 const unsigned WiderElementSize = ElementSize * 2;
17342 if (WiderElementSize > ST.getELen()/8)
17345 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
17348 for (
unsigned i = 0; i < Index->getNumOperands(); i++) {
17351 if (Index->getOperand(i)->isUndef())
17355 uint64_t C = Index->getConstantOperandVal(i);
17357 if (
C % WiderElementSize != 0)
17362 if (
C !=
Last + ElementSize)
17379 (isa<RegisterSDNode>(VL) &&
17380 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
17382 Mask.getOperand(0) != VL)
17385 auto IsTruncNode = [&](
SDValue V) {
17387 V.getOperand(1) == Mask && V.getOperand(2) == VL;
17394 while (IsTruncNode(
Op)) {
17395 if (!
Op.hasOneUse())
17397 Op =
Op.getOperand(0);
17432 MVT VT =
N->getSimpleValueType(0);
17437 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
17439 if (V.getOpcode() != Opc &&
17440 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
17441 V.getOperand(3) == Mask && V.getOperand(4) == VL))
17449 Op.getOperand(1).getValueType().isFixedLengthVector() &&
17451 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
17453 Op =
Op.getOperand(1).getOperand(0);
17456 return V.getOperand(0);
17459 Op.getOperand(2) == VL) {
17460 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
17462 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
17463 return V.getOperand(0);
17472 auto DetectUSatPattern = [&](
SDValue V) {
17497 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
17503 auto DetectSSatPattern = [&](
SDValue V) {
17505 unsigned NumSrcBits = V.getScalarValueSizeInBits();
17513 if (HiC == SignedMax && LoC == SignedMin)
17519 if (HiC == SignedMax && LoC == SignedMin)
17529 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
17531 Src = Src.getOperand(0);
17535 if ((Val = DetectUSatPattern(Src)))
17537 else if ((Val = DetectSSatPattern(Src)))
17547 Val = DAG.
getNode(ClipOpc,
DL, ValVT, Val, Mask, VL);
17548 }
while (ValVT != VT);
17562 EVT VT =
N->getValueType(0);
17570 Src = Src.getOperand(0);
17575 Src = Src.getOperand(0);
17576 EVT SrcEVT = Src.getValueType();
17608 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
17619 switch (
N->getOpcode()) {
17639 APInt V =
C->getValueAPF().bitcastToAPInt();
17674 if (SimplifyDemandedLowBitsHelper(0, 32) ||
17675 SimplifyDemandedLowBitsHelper(1, 5))
17683 if (SimplifyDemandedLowBitsHelper(0, 32))
17700 MVT VT =
N->getSimpleValueType(0);
17703 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(Op0)) {
17716 "Unexpected value type!");
17721 cast<LoadSDNode>(Op0)->isSimple()) {
17723 auto *LN0 = cast<LoadSDNode>(Op0);
17726 LN0->getBasePtr(), IVT, LN0->getMemOperand());
17750 EVT VT =
N->getValueType(0);
17803 if (!
C || !
C->getValueAPF().isExactlyValue(+1.0))
17805 EVT VT =
N->getValueType(0);
17834 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
17839 Src.getOperand(0));
17844 Src.getOperand(0), Src.getOperand(1));
17865 unsigned Opc =
N->getOpcode();
17880 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
17890 N->getValueType(0), Val,
Cond.getOperand(0));
17901 SDValue FalseV =
N->getOperand(4);
17903 EVT VT =
N->getValueType(0);
17906 if (TrueV == FalseV)
17911 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
17917 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
17918 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
17921 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
17922 isInt<12>(TrueSImm - FalseSImm)) {
17938 {LHS, RHS, CC, TrueV, FalseV});
18005 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
18018 EVT VT =
N->getValueType(0);
18042 const auto *MGN = cast<MaskedGatherSDNode>(
N);
18043 const EVT VT =
N->getValueType(0);
18044 SDValue Index = MGN->getIndex();
18045 SDValue ScaleOp = MGN->getScale();
18047 assert(!MGN->isIndexScaled() &&
18048 "Scaled gather/scatter should not be formed");
18053 N->getVTList(), MGN->getMemoryVT(),
DL,
18054 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
18055 MGN->getBasePtr(), Index, ScaleOp},
18056 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
18060 N->getVTList(), MGN->getMemoryVT(),
DL,
18061 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
18062 MGN->getBasePtr(), Index, ScaleOp},
18063 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
18069 if (std::optional<VIDSequence> SimpleVID =
18071 SimpleVID && SimpleVID->StepDenominator == 1) {
18072 const int64_t StepNumerator = SimpleVID->StepNumerator;
18073 const int64_t Addend = SimpleVID->Addend;
18080 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
18088 VT,
DL, MGN->getChain(), BasePtr,
18090 EVL, MGN->getMemOperand());
18092 StridedLoad, MGN->getPassThru(), EVL);
18102 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
18104 MGN->getMemoryVT(), MGN->getMemOperand(),
18113 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
18115 for (
unsigned i = 0; i < Index->getNumOperands(); i += 2)
18116 NewIndices.
push_back(Index.getOperand(i));
18117 EVT IndexVT = Index.getValueType()
18118 .getHalfNumVectorElementsVT(*DAG.
getContext());
18124 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
18126 EltCnt.divideCoefficientBy(2));
18129 EltCnt.divideCoefficientBy(2));
18134 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
18143 const auto *MSN = cast<MaskedScatterSDNode>(
N);
18144 SDValue Index = MSN->getIndex();
18145 SDValue ScaleOp = MSN->getScale();
18147 assert(!MSN->isIndexScaled() &&
18148 "Scaled gather/scatter should not be formed");
18153 N->getVTList(), MSN->getMemoryVT(),
DL,
18154 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
18156 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
18160 N->getVTList(), MSN->getMemoryVT(),
DL,
18161 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
18163 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
18165 EVT VT = MSN->getValue()->getValueType(0);
18167 if (!MSN->isTruncatingStore() &&
18171 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
18172 DAG.
getUNDEF(XLenVT), MSN->getMask(),
18173 MSN->getMemoryVT(), MSN->getMemOperand(),
18178 case ISD::VP_GATHER: {
18179 const auto *VPGN = cast<VPGatherSDNode>(
N);
18180 SDValue Index = VPGN->getIndex();
18181 SDValue ScaleOp = VPGN->getScale();
18183 assert(!VPGN->isIndexScaled() &&
18184 "Scaled gather/scatter should not be formed");
18189 {VPGN->getChain(), VPGN->getBasePtr(), Index,
18190 ScaleOp, VPGN->getMask(),
18191 VPGN->getVectorLength()},
18192 VPGN->getMemOperand(), IndexType);
18196 {VPGN->getChain(), VPGN->getBasePtr(), Index,
18197 ScaleOp, VPGN->getMask(),
18198 VPGN->getVectorLength()},
18199 VPGN->getMemOperand(), IndexType);
18203 case ISD::VP_SCATTER: {
18204 const auto *VPSN = cast<VPScatterSDNode>(
N);
18205 SDValue Index = VPSN->getIndex();
18206 SDValue ScaleOp = VPSN->getScale();
18208 assert(!VPSN->isIndexScaled() &&
18209 "Scaled gather/scatter should not be formed");
18214 {VPSN->getChain(), VPSN->getValue(),
18215 VPSN->getBasePtr(), Index, ScaleOp,
18216 VPSN->getMask(), VPSN->getVectorLength()},
18217 VPSN->getMemOperand(), IndexType);
18221 {VPSN->getChain(), VPSN->getValue(),
18222 VPSN->getBasePtr(), Index, ScaleOp,
18223 VPSN->getMask(), VPSN->getVectorLength()},
18224 VPSN->getMemOperand(), IndexType);
18238 EVT VT =
N->getValueType(0);
18241 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
18242 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
18260 EVT VT =
N->getValueType(0);
18264 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
18304 auto *Store = cast<StoreSDNode>(
N);
18305 SDValue Chain = Store->getChain();
18306 EVT MemVT = Store->getMemoryVT();
18307 SDValue Val = Store->getValue();
18310 bool IsScalarizable =
18312 Store->isSimple() &&
18342 NewVT, *Store->getMemOperand())) {
18344 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
18345 Store->getPointerInfo(), Store->getOriginalAlign(),
18346 Store->getMemOperand()->getFlags());
18354 if (
auto *L = dyn_cast<LoadSDNode>(Val);
18356 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
18358 L->getMemoryVT() == MemVT) {
18361 NewVT, *Store->getMemOperand()) &&
18363 NewVT, *L->getMemOperand())) {
18365 L->getPointerInfo(), L->getOriginalAlign(),
18366 L->getMemOperand()->getFlags());
18367 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
18368 Store->getPointerInfo(), Store->getOriginalAlign(),
18369 Store->getMemOperand()->getFlags());
18381 MVT VecVT = Src.getSimpleValueType();
18388 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
18391 Store->getMemOperand(), Store->getAddressingMode(),
18392 Store->isTruncatingStore(),
false);
18399 EVT VT =
N->getValueType(0);
18425 const MVT VT =
N->getSimpleValueType(0);
18426 SDValue Passthru =
N->getOperand(0);
18427 SDValue Scalar =
N->getOperand(1);
18436 const MVT VT =
N->getSimpleValueType(0);
18437 SDValue Passthru =
N->getOperand(0);
18438 SDValue Scalar =
N->getOperand(1);
18443 unsigned ScalarSize = Scalar.getValueSizeInBits();
18445 if (ScalarSize > EltWidth && Passthru.
isUndef())
18446 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
18453 (!Const || Const->isZero() ||
18454 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
18464 if (
N->getOperand(0).isUndef() &&
18467 Src.getOperand(0).getValueType().isScalableVector()) {
18468 EVT VT =
N->getValueType(0);
18469 EVT SrcVT = Src.getOperand(0).getValueType();
18473 return Src.getOperand(0);
18479 const MVT VT =
N->getSimpleValueType(0);
18480 SDValue Passthru =
N->getOperand(0);
18481 SDValue Scalar =
N->getOperand(1);
18485 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
18486 return Scalar.getOperand(0);
18495 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
18505 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
18513 MVT VecVT =
N->getOperand(0).getSimpleValueType();
18515 if (M1VT.
bitsLT(VecVT)) {
18526 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
18531 case Intrinsic::riscv_vcpop:
18532 case Intrinsic::riscv_vcpop_mask:
18533 case Intrinsic::riscv_vfirst:
18534 case Intrinsic::riscv_vfirst_mask: {
18536 if (IntNo == Intrinsic::riscv_vcpop_mask ||
18537 IntNo == Intrinsic::riscv_vfirst_mask)
18538 VL =
N->getOperand(3);
18543 EVT VT =
N->getValueType(0);
18544 if (IntNo == Intrinsic::riscv_vfirst ||
18545 IntNo == Intrinsic::riscv_vfirst_mask)
18551 case ISD::EXPERIMENTAL_VP_REVERSE:
18553 case ISD::VP_STORE:
18558 EVT VT =
N->getValueType(0);
18569 for (
unsigned i = 0; i < NF; ++i)
18576 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
18599 EVT XVT,
unsigned KeptBits)
const {
18604 if (XVT != MVT::i32 && XVT != MVT::i64)
18608 if (KeptBits == 32 || KeptBits == 64)
18612 return Subtarget.hasStdExtZbb() &&
18613 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
18621 "Expected shift op");
18639 if (!isa<StoreSDNode>(
Use) && !isa<LoadSDNode>(
Use))
18648 return isUsedByLdSt(N0.
getNode(),
N);
18650 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
18651 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
18654 if (Subtarget.hasStdExtZba() && C2 && C2->getZExtValue() >= 1 &&
18655 C2->getZExtValue() <= 3 &&
N->hasOneUse() &&
18656 N->user_begin()->getOpcode() ==
ISD::ADD &&
18657 !isUsedByLdSt(*
N->user_begin(),
nullptr) &&
18658 !isa<ConstantSDNode>(
N->user_begin()->getOperand(1)))
18662 const APInt &C1Int = C1->getAPIntValue();
18663 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
18689 if (C1Cost < ShiftedC1Cost)
18712 EVT VT =
Op.getValueType();
18716 unsigned Opcode =
Op.getOpcode();
18724 const APInt &Mask =
C->getAPIntValue();
18733 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
18734 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
18736 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
18737 if (NewMask == Mask)
18742 Op.getOperand(0), NewC);
18755 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
18756 if (IsLegalMask(NewMask))
18757 return UseMask(NewMask);
18760 if (VT == MVT::i64) {
18762 if (IsLegalMask(NewMask))
18763 return UseMask(NewMask);
18778 APInt NewMask = ShrunkMask;
18779 if (MinSignedBits <= 12)
18781 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
18787 assert(IsLegalMask(NewMask));
18788 return UseMask(NewMask);
18792 static const uint64_t GREVMasks[] = {
18793 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
18794 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
18796 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
18797 unsigned Shift = 1 << Stage;
18798 if (ShAmt & Shift) {
18800 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
18812 const APInt &DemandedElts,
18814 unsigned Depth)
const {
18816 unsigned Opc =
Op.getOpcode();
18821 "Should use MaskedValueIsZero if you don't know whether Op"
18822 " is a target node!");
18905 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
18908 if (MaxVLenB == MinVLenB)
18925 case Intrinsic::riscv_vsetvli:
18926 case Intrinsic::riscv_vsetvlimax: {
18927 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
18928 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
18934 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
18937 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
18938 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
18940 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
18953 unsigned Depth)
const {
18954 switch (
Op.getOpcode()) {
18960 if (Tmp == 1)
return 1;
18963 return std::min(Tmp, Tmp2);
18975 if (Tmp < 33)
return 1;
19000 unsigned XLen = Subtarget.
getXLen();
19001 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
19002 if (EltBits <= XLen)
19003 return XLen - EltBits + 1;
19007 unsigned IntNo =
Op.getConstantOperandVal(1);
19011 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
19012 case Intrinsic::riscv_masked_atomicrmw_add_i64:
19013 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
19014 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
19015 case Intrinsic::riscv_masked_atomicrmw_max_i64:
19016 case Intrinsic::riscv_masked_atomicrmw_min_i64:
19017 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
19018 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
19019 case Intrinsic::riscv_masked_cmpxchg_i64:
19027 assert(Subtarget.hasStdExtA());
19042 switch (
Op.getOpcode()) {
19048 return !
Op.getValueType().isInteger();
19056 assert(Ld &&
"Unexpected null LoadSDNode");
19064 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
19065 if (!CNode || CNode->isMachineConstantPoolEntry() ||
19066 CNode->getOffset() != 0)
19074 auto *CNode = GetSupportedConstantPool(
Ptr);
19075 if (!CNode || CNode->getTargetFlags() != 0)
19078 return CNode->getConstVal();
19086 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
19087 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
19093 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
19096 return CNodeLo->getConstVal();
19101 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
19133 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19136 int64_t LoCounter =
MI.getOperand(2).getImm();
19137 int64_t HiCounter =
MI.getOperand(3).getImm();
19147 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
19159 MI.eraseFromParent();
19167 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
19175 Register SrcReg =
MI.getOperand(2).getReg();
19195 MI.eraseFromParent();
19202 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
19203 "Unexpected instruction");
19209 Register DstReg =
MI.getOperand(0).getReg();
19232 MI.eraseFromParent();
19237 switch (
MI.getOpcode()) {
19240 case RISCV::Select_GPR_Using_CC_GPR:
19241 case RISCV::Select_GPR_Using_CC_Imm:
19242 case RISCV::Select_FPR16_Using_CC_GPR:
19243 case RISCV::Select_FPR16INX_Using_CC_GPR:
19244 case RISCV::Select_FPR32_Using_CC_GPR:
19245 case RISCV::Select_FPR32INX_Using_CC_GPR:
19246 case RISCV::Select_FPR64_Using_CC_GPR:
19247 case RISCV::Select_FPR64INX_Using_CC_GPR:
19248 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
19254 unsigned RelOpcode,
unsigned EqOpcode,
19257 Register DstReg =
MI.getOperand(0).getReg();
19258 Register Src1Reg =
MI.getOperand(1).getReg();
19259 Register Src2Reg =
MI.getOperand(2).getReg();
19261 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19285 MI.eraseFromParent();
19336 F->insert(It, FirstMBB);
19337 F->insert(It, SecondMBB);
19338 F->insert(It, SinkMBB);
19387 First.eraseFromParent();
19426 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
19427 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
19428 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
19429 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
19430 Next->getOperand(5).isKill())
19435 if (
MI.getOperand(2).isReg())
19436 RHS =
MI.getOperand(2).getReg();
19441 SelectDests.
insert(
MI.getOperand(0).getReg());
19445 SequenceMBBI != E; ++SequenceMBBI) {
19446 if (SequenceMBBI->isDebugInstr())
19449 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
19450 !SequenceMBBI->getOperand(2).isReg() ||
19451 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
19452 SequenceMBBI->getOperand(3).getImm() !=
CC ||
19453 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
19454 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
19456 LastSelectPseudo = &*SequenceMBBI;
19458 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
19461 if (SequenceMBBI->hasUnmodeledSideEffects() ||
19462 SequenceMBBI->mayLoadOrStore() ||
19463 SequenceMBBI->usesCustomInsertionHook())
19466 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
19481 F->insert(
I, IfFalseMBB);
19482 F->insert(
I, TailMBB);
19485 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
19491 TailMBB->
push_back(DebugInstr->removeFromParent());
19495 TailMBB->
splice(TailMBB->
end(), HeadMBB,
19505 if (
MI.getOperand(2).isImm())
19508 .
addImm(
MI.getOperand(2).getImm())
19520 auto SelectMBBI =
MI.getIterator();
19521 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
19523 while (SelectMBBI != SelectEnd) {
19524 auto Next = std::next(SelectMBBI);
19528 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
19529 .
addReg(SelectMBBI->getOperand(4).getReg())
19531 .
addReg(SelectMBBI->getOperand(5).getReg())
19546 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
19547 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
19549 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
19550 assert(
Masked &&
"Could not find masked instruction for LMUL and SEW pair");
19556 unsigned CVTXOpc) {
19562 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19575 .
add(
MI.getOperand(1))
19576 .
add(
MI.getOperand(2))
19577 .
add(
MI.getOperand(3))
19579 .
add(
MI.getOperand(4))
19580 .
add(
MI.getOperand(5))
19581 .
add(
MI.getOperand(6))
19596 .
add(
MI.getOperand(0))
19597 .
add(
MI.getOperand(1))
19599 .
add(
MI.getOperand(3))
19601 .
add(
MI.getOperand(4))
19602 .
add(
MI.getOperand(5))
19603 .
add(
MI.getOperand(6))
19613 MI.eraseFromParent();
19619 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
19621 switch (
MI.getOpcode()) {
19624 case RISCV::PseudoFROUND_H:
19625 CmpOpc = RISCV::FLT_H;
19626 F2IOpc = RISCV::FCVT_W_H;
19627 I2FOpc = RISCV::FCVT_H_W;
19628 FSGNJOpc = RISCV::FSGNJ_H;
19629 FSGNJXOpc = RISCV::FSGNJX_H;
19630 RC = &RISCV::FPR16RegClass;
19632 case RISCV::PseudoFROUND_H_INX:
19633 CmpOpc = RISCV::FLT_H_INX;
19634 F2IOpc = RISCV::FCVT_W_H_INX;
19635 I2FOpc = RISCV::FCVT_H_W_INX;
19636 FSGNJOpc = RISCV::FSGNJ_H_INX;
19637 FSGNJXOpc = RISCV::FSGNJX_H_INX;
19638 RC = &RISCV::GPRF16RegClass;
19640 case RISCV::PseudoFROUND_S:
19641 CmpOpc = RISCV::FLT_S;
19642 F2IOpc = RISCV::FCVT_W_S;
19643 I2FOpc = RISCV::FCVT_S_W;
19644 FSGNJOpc = RISCV::FSGNJ_S;
19645 FSGNJXOpc = RISCV::FSGNJX_S;
19646 RC = &RISCV::FPR32RegClass;
19648 case RISCV::PseudoFROUND_S_INX:
19649 CmpOpc = RISCV::FLT_S_INX;
19650 F2IOpc = RISCV::FCVT_W_S_INX;
19651 I2FOpc = RISCV::FCVT_S_W_INX;
19652 FSGNJOpc = RISCV::FSGNJ_S_INX;
19653 FSGNJXOpc = RISCV::FSGNJX_S_INX;
19654 RC = &RISCV::GPRF32RegClass;
19656 case RISCV::PseudoFROUND_D:
19658 CmpOpc = RISCV::FLT_D;
19659 F2IOpc = RISCV::FCVT_L_D;
19660 I2FOpc = RISCV::FCVT_D_L;
19661 FSGNJOpc = RISCV::FSGNJ_D;
19662 FSGNJXOpc = RISCV::FSGNJX_D;
19663 RC = &RISCV::FPR64RegClass;
19665 case RISCV::PseudoFROUND_D_INX:
19667 CmpOpc = RISCV::FLT_D_INX;
19668 F2IOpc = RISCV::FCVT_L_D_INX;
19669 I2FOpc = RISCV::FCVT_D_L_INX;
19670 FSGNJOpc = RISCV::FSGNJ_D_INX;
19671 FSGNJXOpc = RISCV::FSGNJX_D_INX;
19672 RC = &RISCV::GPRRegClass;
19684 F->insert(
I, CvtMBB);
19685 F->insert(
I, DoneMBB);
19696 Register DstReg =
MI.getOperand(0).getReg();
19697 Register SrcReg =
MI.getOperand(1).getReg();
19698 Register MaxReg =
MI.getOperand(2).getReg();
19699 int64_t FRM =
MI.getOperand(3).getImm();
19704 Register FabsReg =
MRI.createVirtualRegister(RC);
19708 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19723 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19745 MI.eraseFromParent();
19752 switch (
MI.getOpcode()) {
19755 case RISCV::ReadCounterWide:
19757 "ReadCounterWide is only to be used on riscv32");
19759 case RISCV::Select_GPR_Using_CC_GPR:
19760 case RISCV::Select_GPR_Using_CC_Imm:
19761 case RISCV::Select_FPR16_Using_CC_GPR:
19762 case RISCV::Select_FPR16INX_Using_CC_GPR:
19763 case RISCV::Select_FPR32_Using_CC_GPR:
19764 case RISCV::Select_FPR32INX_Using_CC_GPR:
19765 case RISCV::Select_FPR64_Using_CC_GPR:
19766 case RISCV::Select_FPR64INX_Using_CC_GPR:
19767 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
19769 case RISCV::BuildPairF64Pseudo:
19771 case RISCV::SplitF64Pseudo:
19773 case RISCV::PseudoQuietFLE_H:
19775 case RISCV::PseudoQuietFLE_H_INX:
19776 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
19777 case RISCV::PseudoQuietFLT_H:
19779 case RISCV::PseudoQuietFLT_H_INX:
19780 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
19781 case RISCV::PseudoQuietFLE_S:
19783 case RISCV::PseudoQuietFLE_S_INX:
19784 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
19785 case RISCV::PseudoQuietFLT_S:
19787 case RISCV::PseudoQuietFLT_S_INX:
19788 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
19789 case RISCV::PseudoQuietFLE_D:
19791 case RISCV::PseudoQuietFLE_D_INX:
19792 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
19793 case RISCV::PseudoQuietFLE_D_IN32X:
19796 case RISCV::PseudoQuietFLT_D:
19798 case RISCV::PseudoQuietFLT_D_INX:
19799 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
19800 case RISCV::PseudoQuietFLT_D_IN32X:
19804 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
19806 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
19808 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
19810 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
19812 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
19814 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
19816 case RISCV::PseudoFROUND_H:
19817 case RISCV::PseudoFROUND_H_INX:
19818 case RISCV::PseudoFROUND_S:
19819 case RISCV::PseudoFROUND_S_INX:
19820 case RISCV::PseudoFROUND_D:
19821 case RISCV::PseudoFROUND_D_INX:
19822 case RISCV::PseudoFROUND_D_IN32X:
19824 case RISCV::PROBED_STACKALLOC_DYN:
19826 case TargetOpcode::STATEPOINT:
19832 MI.addOperand(*
MI.getMF(),
19838 case TargetOpcode::STACKMAP:
19839 case TargetOpcode::PATCHPOINT:
19842 "supported on 64-bit targets");
19860 if (
MI.readsRegister(RISCV::FRM,
nullptr))
19866void RISCVTargetLowering::analyzeInputArgs(
19870 unsigned NumArgs = Ins.size();
19873 for (
unsigned i = 0; i != NumArgs; ++i) {
19874 MVT ArgVT = Ins[i].VT;
19877 Type *ArgTy =
nullptr;
19880 else if (Ins[i].isOrigArg())
19881 ArgTy = FType->
getParamType(Ins[i].getOrigArgIndex());
19884 true, IsRet, ArgTy)) {
19885 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
19892void RISCVTargetLowering::analyzeOutputArgs(
19896 unsigned NumArgs = Outs.
size();
19898 for (
unsigned i = 0; i != NumArgs; i++) {
19899 MVT ArgVT = Outs[i].VT;
19901 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
19904 Outs[i].IsFixed, IsRet, OrigTy)) {
19905 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
19956 if (In.isOrigArg()) {
19961 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
19962 (
BitWidth < 32 && In.Flags.isZExt())) {
19984 if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32)
20033 ExtType,
DL, LocVT, Chain, FIN,
20050 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
20063 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
20078 switch (CallConv) {
20088 if (Subtarget.hasStdExtE())
20092 "(Zdinx/D) instruction set extensions");
20096 if (Func.hasFnAttribute(
"interrupt")) {
20097 if (!Func.arg_empty())
20099 "Functions with the interrupt attribute cannot have arguments!");
20104 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
20106 "Function interrupt attribute argument not supported!");
20111 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
20113 std::vector<SDValue> OutChains;
20122 analyzeInputArgs(MF, CCInfo, Ins,
false,
20126 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
20147 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
20148 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
20150 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
20152 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
20181 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
20186 if (VarArgsSaveSize == 0) {
20190 int VaArgOffset = -VarArgsSaveSize;
20198 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
20199 VarArgsSaveSize += XLenInBytes;
20206 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
20211 Chain,
DL, ArgValue, FIN,
20213 OutChains.push_back(Store);
20227 if (!OutChains.empty()) {
20228 OutChains.push_back(Chain);
20238bool RISCVTargetLowering::isEligibleForTailCallOptimization(
20242 auto CalleeCC = CLI.CallConv;
20243 auto &Outs = CLI.Outs;
20245 auto CallerCC = Caller.getCallingConv();
20252 if (Caller.hasFnAttribute(
"interrupt"))
20267 for (
auto &VA : ArgLocs)
20273 auto IsCallerStructRet = Caller.hasStructRetAttr();
20274 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
20275 if (IsCallerStructRet || IsCalleeStructRet)
20280 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
20281 if (CalleeCC != CallerCC) {
20282 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
20283 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
20290 for (
auto &Arg : Outs)
20291 if (Arg.Flags.isByVal())
20326 if (Subtarget.hasStdExtE())
20330 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
20336 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
20342 "site marked musttail");
20349 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20351 if (!Flags.isByVal())
20355 unsigned Size = Flags.getByValSize();
20356 Align Alignment = Flags.getNonZeroByValAlign();
20363 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
20365 false,
nullptr, IsTailCall,
20377 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
20380 SDValue ArgValue = OutVals[OutIdx];
20400 if (!StackPtr.getNode())
20412 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
20430 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
20431 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
20437 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
20438 SDValue PartValue = OutVals[OutIdx + 1];
20439 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
20451 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
20453 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
20455 for (
const auto &Part : Parts) {
20456 SDValue PartValue = Part.first;
20457 SDValue PartOffset = Part.second;
20464 ArgValue = SpillSlot;
20470 if (Flags.isByVal())
20471 ArgValue = ByValArgs[j++];
20478 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
20479 "for passing parameters");
20482 if (!StackPtr.getNode())
20496 if (!MemOpChains.
empty())
20502 for (
auto &Reg : RegsToPass) {
20503 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
20510 validateCCReservedRegs(RegsToPass, MF);
20514 "Return address register required, but has been reserved."});
20519 bool CalleeIsLargeExternalSymbol =
false;
20521 if (
auto *S = dyn_cast<GlobalAddressSDNode>(Callee))
20523 else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
20525 CalleeIsLargeExternalSymbol =
true;
20541 for (
auto &Reg : RegsToPass)
20546 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
20547 assert(Mask &&
"Missing call preserved mask for calling convention");
20555 "Unexpected CFI type for a direct call");
20563 bool NeedSWGuarded =
false;
20565 Subtarget.hasStdExtZicfilp() &&
20567 NeedSWGuarded =
true;
20581 Chain = DAG.
getNode(CallOpc,
DL, NodeTys, Ops);
20594 analyzeInputArgs(MF, RetCCInfo, Ins,
true,
CC_RISCV);
20597 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
20598 auto &VA = RVLocs[i];
20606 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
20607 assert(VA.needsCustom());
20628 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
20630 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20631 MVT VT = Outs[i].VT;
20634 true,
true,
nullptr))
20666 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
20667 SDValue Val = OutVals[OutIdx];
20676 DAG.
getVTList(MVT::i32, MVT::i32), Val);
20680 Register RegHi = RVLocs[++i].getLocReg();
20686 "Return value register required, but has been reserved."});
20702 "Return value register required, but has been reserved."});
20724 if (Func.hasFnAttribute(
"interrupt")) {
20725 if (!Func.getReturnType()->isVoidTy())
20727 "Functions with the interrupt attribute must have void return type!");
20733 if (Kind ==
"supervisor")
20739 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
20742void RISCVTargetLowering::validateCCReservedRegs(
20743 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
20752 F,
"Argument register required, but has been reserved."});
20758 if (
N->getNumValues() != 1)
20760 if (!
N->hasNUsesOfValue(1, 0))
20763 SDNode *Copy = *
N->user_begin();
20777 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20781 bool HasRet =
false;
20782 for (
SDNode *Node : Copy->users()) {
20790 Chain = Copy->getOperand(0);
20799#define NODE_NAME_CASE(NODE) \
20800 case RISCVISD::NODE: \
20801 return "RISCVISD::" #NODE;