38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
274 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
286 if (!Subtarget.hasStdExtZbb())
290 if (Subtarget.hasStdExtZbb()) {
298 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
303 if (!Subtarget.hasStdExtZmmul()) {
307 }
else if (Subtarget.
is64Bit()) {
317 if (!Subtarget.hasStdExtM()) {
323 }
else if (Subtarget.
is64Bit()) {
326 {MVT::i8, MVT::i16, MVT::i32},
Custom);
343 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
346 }
else if (Subtarget.hasVendorXTHeadBb()) {
350 }
else if (Subtarget.hasVendorXCVbitmanip()) {
361 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
362 Subtarget.hasVendorXTHeadBb())
367 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
368 Subtarget.hasVendorXTHeadBb())
373 if (Subtarget.hasVendorXCVbitmanip()) {
381 if (Subtarget.hasStdExtZbb()) {
394 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
400 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
401 Subtarget.hasVendorXCVbitmanip()) {
408 if (!Subtarget.hasStdExtZbb())
420 !Subtarget.hasShortForwardBranchOpt())
424 if (Subtarget.hasShortForwardBranchOpt())
427 if (!Subtarget.hasVendorXTHeadCondMov()) {
433 static const unsigned FPLegalNodeTypes[] = {
446 static const unsigned FPOpToExpand[] = {
450 static const unsigned FPRndMode[] = {
457 static const unsigned ZfhminZfbfminPromoteOps[] = {
468 if (Subtarget.hasStdExtZfbfmin()) {
551 if (Subtarget.hasStdExtZfa()) {
568 if (Subtarget.hasStdExtZfa()) {
645 if (Subtarget.hasStdExtZicbop()) {
649 if (Subtarget.hasStdExtA()) {
651 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
655 }
else if (Subtarget.hasForcedAtomics()) {
681 {MVT::i8, MVT::i16},
Custom);
692 static const unsigned IntegerVPOps[] = {
693 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
694 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
695 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
696 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
697 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
698 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
699 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
700 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
701 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
702 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
703 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
704 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
705 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
706 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF};
708 static const unsigned FloatingPointVPOps[] = {
709 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
710 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
711 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
712 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
713 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
714 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
715 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
716 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
717 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
718 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
719 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
720 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
721 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
722 ISD::VP_REDUCE_FMAXIMUM};
724 static const unsigned IntegerVecReduceOps[] = {
729 static const unsigned FloatingPointVecReduceOps[] = {
742 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
743 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
744 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
748 for (
MVT VT : BoolVecVTs) {
778 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
802 ISD::VP_TRUNCATE, ISD::VP_SETCC},
818 for (
MVT VT : IntVecVTs) {
829 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
877 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
878 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
902 if (Subtarget.hasStdExtZvkb()) {
910 if (Subtarget.hasStdExtZvbb()) {
914 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
920 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
929 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
949 static const unsigned ZvfhminPromoteOps[] = {
959 static const unsigned ZvfhminPromoteVPOps[] = {
960 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
961 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
962 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
963 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
964 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
965 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
966 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
967 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
968 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
971 const auto SetCommonVFPActions = [&](
MVT VT) {
1016 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1017 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1048 const auto SetCommonVFPExtLoadTruncStoreActions =
1050 for (
auto SmallVT : SmallerVTs) {
1057 for (
MVT VT : F16VecVTs) {
1060 SetCommonVFPActions(VT);
1063 for (
MVT VT : F16VecVTs) {
1074 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1085 if (VT == MVT::nxv32f16) {
1099 for (
MVT VT : BF16VecVTs) {
1110 if (Subtarget.hasStdExtZfbfmin())
1120 for (
MVT VT : F32VecVTs) {
1123 SetCommonVFPActions(VT);
1124 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1129 for (
MVT VT : F64VecVTs) {
1132 SetCommonVFPActions(VT);
1133 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1134 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1140 if (!useRVVForFixedLengthVectorVT(VT))
1183 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1210 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1232 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1233 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1271 if (Subtarget.hasStdExtZvkb())
1274 if (Subtarget.hasStdExtZvbb()) {
1296 if (!useRVVForFixedLengthVectorVT(VT))
1317 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1344 if (Subtarget.hasStdExtZfbfmin())
1367 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1368 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1419 if (Subtarget.hasStdExtA()) {
1425 if (Subtarget.hasForcedAtomics()) {
1435 if (Subtarget.hasVendorXTHeadMemIdx()) {
1451 if (Subtarget.hasVendorXCVmem()) {
1477 if (Subtarget.hasStdExtZbb())
1480 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1481 Subtarget.hasStdExtV())
1484 if (Subtarget.hasStdExtZbkb())
1496 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1499 if (Subtarget.hasVendorXTHeadMemPair())
1522MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1527bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1529 bool IsScalable)
const {
1536 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1561 unsigned Intrinsic)
const {
1562 auto &
DL =
I.getDataLayout();
1564 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1565 bool IsUnitStrided,
bool UsePtrVal =
false) {
1570 Info.ptrVal =
I.getArgOperand(PtrOp);
1572 Info.fallbackAddressSpace =
1573 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1577 MemTy =
I.getArgOperand(0)->getType();
1580 MemTy =
I.getType();
1595 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1599 switch (Intrinsic) {
1602 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1603 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1604 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1605 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1606 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1607 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1608 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1609 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1610 case Intrinsic::riscv_masked_cmpxchg_i32:
1612 Info.memVT = MVT::i32;
1613 Info.ptrVal =
I.getArgOperand(0);
1619 case Intrinsic::riscv_masked_strided_load:
1620 return SetRVVLoadStoreInfo( 1,
false,
1622 case Intrinsic::riscv_masked_strided_store:
1623 return SetRVVLoadStoreInfo( 1,
true,
1625 case Intrinsic::riscv_seg2_load:
1626 case Intrinsic::riscv_seg3_load:
1627 case Intrinsic::riscv_seg4_load:
1628 case Intrinsic::riscv_seg5_load:
1629 case Intrinsic::riscv_seg6_load:
1630 case Intrinsic::riscv_seg7_load:
1631 case Intrinsic::riscv_seg8_load:
1632 return SetRVVLoadStoreInfo( 0,
false,
1634 case Intrinsic::riscv_seg2_store:
1635 case Intrinsic::riscv_seg3_store:
1636 case Intrinsic::riscv_seg4_store:
1637 case Intrinsic::riscv_seg5_store:
1638 case Intrinsic::riscv_seg6_store:
1639 case Intrinsic::riscv_seg7_store:
1640 case Intrinsic::riscv_seg8_store:
1642 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1645 case Intrinsic::riscv_vle:
1646 case Intrinsic::riscv_vle_mask:
1647 case Intrinsic::riscv_vleff:
1648 case Intrinsic::riscv_vleff_mask:
1649 return SetRVVLoadStoreInfo( 1,
1653 case Intrinsic::riscv_vse:
1654 case Intrinsic::riscv_vse_mask:
1655 return SetRVVLoadStoreInfo( 1,
1659 case Intrinsic::riscv_vlse:
1660 case Intrinsic::riscv_vlse_mask:
1661 case Intrinsic::riscv_vloxei:
1662 case Intrinsic::riscv_vloxei_mask:
1663 case Intrinsic::riscv_vluxei:
1664 case Intrinsic::riscv_vluxei_mask:
1665 return SetRVVLoadStoreInfo( 1,
1668 case Intrinsic::riscv_vsse:
1669 case Intrinsic::riscv_vsse_mask:
1670 case Intrinsic::riscv_vsoxei:
1671 case Intrinsic::riscv_vsoxei_mask:
1672 case Intrinsic::riscv_vsuxei:
1673 case Intrinsic::riscv_vsuxei_mask:
1674 return SetRVVLoadStoreInfo( 1,
1677 case Intrinsic::riscv_vlseg2:
1678 case Intrinsic::riscv_vlseg3:
1679 case Intrinsic::riscv_vlseg4:
1680 case Intrinsic::riscv_vlseg5:
1681 case Intrinsic::riscv_vlseg6:
1682 case Intrinsic::riscv_vlseg7:
1683 case Intrinsic::riscv_vlseg8:
1684 case Intrinsic::riscv_vlseg2ff:
1685 case Intrinsic::riscv_vlseg3ff:
1686 case Intrinsic::riscv_vlseg4ff:
1687 case Intrinsic::riscv_vlseg5ff:
1688 case Intrinsic::riscv_vlseg6ff:
1689 case Intrinsic::riscv_vlseg7ff:
1690 case Intrinsic::riscv_vlseg8ff:
1691 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1694 case Intrinsic::riscv_vlseg2_mask:
1695 case Intrinsic::riscv_vlseg3_mask:
1696 case Intrinsic::riscv_vlseg4_mask:
1697 case Intrinsic::riscv_vlseg5_mask:
1698 case Intrinsic::riscv_vlseg6_mask:
1699 case Intrinsic::riscv_vlseg7_mask:
1700 case Intrinsic::riscv_vlseg8_mask:
1701 case Intrinsic::riscv_vlseg2ff_mask:
1702 case Intrinsic::riscv_vlseg3ff_mask:
1703 case Intrinsic::riscv_vlseg4ff_mask:
1704 case Intrinsic::riscv_vlseg5ff_mask:
1705 case Intrinsic::riscv_vlseg6ff_mask:
1706 case Intrinsic::riscv_vlseg7ff_mask:
1707 case Intrinsic::riscv_vlseg8ff_mask:
1708 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1711 case Intrinsic::riscv_vlsseg2:
1712 case Intrinsic::riscv_vlsseg3:
1713 case Intrinsic::riscv_vlsseg4:
1714 case Intrinsic::riscv_vlsseg5:
1715 case Intrinsic::riscv_vlsseg6:
1716 case Intrinsic::riscv_vlsseg7:
1717 case Intrinsic::riscv_vlsseg8:
1718 case Intrinsic::riscv_vloxseg2:
1719 case Intrinsic::riscv_vloxseg3:
1720 case Intrinsic::riscv_vloxseg4:
1721 case Intrinsic::riscv_vloxseg5:
1722 case Intrinsic::riscv_vloxseg6:
1723 case Intrinsic::riscv_vloxseg7:
1724 case Intrinsic::riscv_vloxseg8:
1725 case Intrinsic::riscv_vluxseg2:
1726 case Intrinsic::riscv_vluxseg3:
1727 case Intrinsic::riscv_vluxseg4:
1728 case Intrinsic::riscv_vluxseg5:
1729 case Intrinsic::riscv_vluxseg6:
1730 case Intrinsic::riscv_vluxseg7:
1731 case Intrinsic::riscv_vluxseg8:
1732 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1735 case Intrinsic::riscv_vlsseg2_mask:
1736 case Intrinsic::riscv_vlsseg3_mask:
1737 case Intrinsic::riscv_vlsseg4_mask:
1738 case Intrinsic::riscv_vlsseg5_mask:
1739 case Intrinsic::riscv_vlsseg6_mask:
1740 case Intrinsic::riscv_vlsseg7_mask:
1741 case Intrinsic::riscv_vlsseg8_mask:
1742 case Intrinsic::riscv_vloxseg2_mask:
1743 case Intrinsic::riscv_vloxseg3_mask:
1744 case Intrinsic::riscv_vloxseg4_mask:
1745 case Intrinsic::riscv_vloxseg5_mask:
1746 case Intrinsic::riscv_vloxseg6_mask:
1747 case Intrinsic::riscv_vloxseg7_mask:
1748 case Intrinsic::riscv_vloxseg8_mask:
1749 case Intrinsic::riscv_vluxseg2_mask:
1750 case Intrinsic::riscv_vluxseg3_mask:
1751 case Intrinsic::riscv_vluxseg4_mask:
1752 case Intrinsic::riscv_vluxseg5_mask:
1753 case Intrinsic::riscv_vluxseg6_mask:
1754 case Intrinsic::riscv_vluxseg7_mask:
1755 case Intrinsic::riscv_vluxseg8_mask:
1756 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1759 case Intrinsic::riscv_vsseg2:
1760 case Intrinsic::riscv_vsseg3:
1761 case Intrinsic::riscv_vsseg4:
1762 case Intrinsic::riscv_vsseg5:
1763 case Intrinsic::riscv_vsseg6:
1764 case Intrinsic::riscv_vsseg7:
1765 case Intrinsic::riscv_vsseg8:
1766 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1769 case Intrinsic::riscv_vsseg2_mask:
1770 case Intrinsic::riscv_vsseg3_mask:
1771 case Intrinsic::riscv_vsseg4_mask:
1772 case Intrinsic::riscv_vsseg5_mask:
1773 case Intrinsic::riscv_vsseg6_mask:
1774 case Intrinsic::riscv_vsseg7_mask:
1775 case Intrinsic::riscv_vsseg8_mask:
1776 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1779 case Intrinsic::riscv_vssseg2:
1780 case Intrinsic::riscv_vssseg3:
1781 case Intrinsic::riscv_vssseg4:
1782 case Intrinsic::riscv_vssseg5:
1783 case Intrinsic::riscv_vssseg6:
1784 case Intrinsic::riscv_vssseg7:
1785 case Intrinsic::riscv_vssseg8:
1786 case Intrinsic::riscv_vsoxseg2:
1787 case Intrinsic::riscv_vsoxseg3:
1788 case Intrinsic::riscv_vsoxseg4:
1789 case Intrinsic::riscv_vsoxseg5:
1790 case Intrinsic::riscv_vsoxseg6:
1791 case Intrinsic::riscv_vsoxseg7:
1792 case Intrinsic::riscv_vsoxseg8:
1793 case Intrinsic::riscv_vsuxseg2:
1794 case Intrinsic::riscv_vsuxseg3:
1795 case Intrinsic::riscv_vsuxseg4:
1796 case Intrinsic::riscv_vsuxseg5:
1797 case Intrinsic::riscv_vsuxseg6:
1798 case Intrinsic::riscv_vsuxseg7:
1799 case Intrinsic::riscv_vsuxseg8:
1800 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1803 case Intrinsic::riscv_vssseg2_mask:
1804 case Intrinsic::riscv_vssseg3_mask:
1805 case Intrinsic::riscv_vssseg4_mask:
1806 case Intrinsic::riscv_vssseg5_mask:
1807 case Intrinsic::riscv_vssseg6_mask:
1808 case Intrinsic::riscv_vssseg7_mask:
1809 case Intrinsic::riscv_vssseg8_mask:
1810 case Intrinsic::riscv_vsoxseg2_mask:
1811 case Intrinsic::riscv_vsoxseg3_mask:
1812 case Intrinsic::riscv_vsoxseg4_mask:
1813 case Intrinsic::riscv_vsoxseg5_mask:
1814 case Intrinsic::riscv_vsoxseg6_mask:
1815 case Intrinsic::riscv_vsoxseg7_mask:
1816 case Intrinsic::riscv_vsoxseg8_mask:
1817 case Intrinsic::riscv_vsuxseg2_mask:
1818 case Intrinsic::riscv_vsuxseg3_mask:
1819 case Intrinsic::riscv_vsuxseg4_mask:
1820 case Intrinsic::riscv_vsuxseg5_mask:
1821 case Intrinsic::riscv_vsuxseg6_mask:
1822 case Intrinsic::riscv_vsuxseg7_mask:
1823 case Intrinsic::riscv_vsuxseg8_mask:
1824 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1861 return isInt<12>(Imm);
1865 return isInt<12>(Imm);
1878 return (SrcBits == 64 && DestBits == 32);
1889 return (SrcBits == 64 && DestBits == 32);
1896 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1897 EVT MemVT = LD->getMemoryVT();
1898 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1908 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1916 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1920 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1921 Subtarget.hasVendorXCVbitmanip();
1932 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1937 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1941 EVT VT =
Y.getValueType();
1947 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1948 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
1953 if (Subtarget.hasStdExtZbs())
1954 return X.getValueType().isScalarInteger();
1955 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1957 if (Subtarget.hasVendorXTHeadBs())
1958 return C !=
nullptr;
1960 return C &&
C->getAPIntValue().ule(10);
1980 if (BitSize > Subtarget.
getXLen())
1984 int64_t Val = Imm.getSExtValue();
1992 if (!Subtarget.enableUnalignedScalarMem())
2008 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
2015 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
2019 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2031 case Instruction::Add:
2032 case Instruction::Sub:
2033 case Instruction::Mul:
2034 case Instruction::And:
2035 case Instruction::Or:
2036 case Instruction::Xor:
2037 case Instruction::FAdd:
2038 case Instruction::FSub:
2039 case Instruction::FMul:
2040 case Instruction::FDiv:
2041 case Instruction::ICmp:
2042 case Instruction::FCmp:
2044 case Instruction::Shl:
2045 case Instruction::LShr:
2046 case Instruction::AShr:
2047 case Instruction::UDiv:
2048 case Instruction::SDiv:
2049 case Instruction::URem:
2050 case Instruction::SRem:
2051 case Instruction::Select:
2052 return Operand == 1;
2066 auto *
II = dyn_cast<IntrinsicInst>(
I);
2070 switch (
II->getIntrinsicID()) {
2071 case Intrinsic::fma:
2072 case Intrinsic::vp_fma:
2073 return Operand == 0 || Operand == 1;
2074 case Intrinsic::vp_shl:
2075 case Intrinsic::vp_lshr:
2076 case Intrinsic::vp_ashr:
2077 case Intrinsic::vp_udiv:
2078 case Intrinsic::vp_sdiv:
2079 case Intrinsic::vp_urem:
2080 case Intrinsic::vp_srem:
2081 case Intrinsic::ssub_sat:
2082 case Intrinsic::vp_ssub_sat:
2083 case Intrinsic::usub_sat:
2084 case Intrinsic::vp_usub_sat:
2085 return Operand == 1;
2087 case Intrinsic::vp_add:
2088 case Intrinsic::vp_mul:
2089 case Intrinsic::vp_and:
2090 case Intrinsic::vp_or:
2091 case Intrinsic::vp_xor:
2092 case Intrinsic::vp_fadd:
2093 case Intrinsic::vp_fmul:
2094 case Intrinsic::vp_icmp:
2095 case Intrinsic::vp_fcmp:
2096 case Intrinsic::smin:
2097 case Intrinsic::vp_smin:
2098 case Intrinsic::umin:
2099 case Intrinsic::vp_umin:
2100 case Intrinsic::smax:
2101 case Intrinsic::vp_smax:
2102 case Intrinsic::umax:
2103 case Intrinsic::vp_umax:
2104 case Intrinsic::sadd_sat:
2105 case Intrinsic::vp_sadd_sat:
2106 case Intrinsic::uadd_sat:
2107 case Intrinsic::vp_uadd_sat:
2109 case Intrinsic::vp_sub:
2110 case Intrinsic::vp_fsub:
2111 case Intrinsic::vp_fdiv:
2112 return Operand == 0 || Operand == 1;
2133 if (!Subtarget.sinkSplatOperands())
2136 for (
auto OpIdx :
enumerate(
I->operands())) {
2140 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2142 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2151 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2156 for (
Use &U :
Op->uses()) {
2208 if (!Subtarget.hasStdExtZfa())
2209 return std::make_pair(-1,
false);
2211 bool IsSupportedVT =
false;
2212 if (VT == MVT::f16) {
2213 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2214 }
else if (VT == MVT::f32) {
2215 IsSupportedVT =
true;
2216 }
else if (VT == MVT::f64) {
2217 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2218 IsSupportedVT =
true;
2222 return std::make_pair(-1,
false);
2225 if (
Index < 0 && Imm.isNegative())
2229 return std::make_pair(
Index,
false);
2233 bool ForCodeSize)
const {
2234 bool IsLegalVT =
false;
2237 else if (VT == MVT::f32)
2239 else if (VT == MVT::f64)
2241 else if (VT == MVT::bf16)
2242 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2254 return Imm.isZero();
2258 if (Imm.isNegZero())
2271 unsigned Index)
const {
2284 if (EltVT == MVT::i1)
2297 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2304 if ((ResElts * 2) != SrcElts)
2348 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2350 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2353 IntermediateVT = MVT::i64;
2356 RegisterVT = MVT::i64;
2371 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2377 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2390 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2391 int64_t
C = RHSC->getSExtValue();
2433 switch (KnownSize) {
2461 return RISCV::VRRegClassID;
2463 return RISCV::VRM2RegClassID;
2465 return RISCV::VRM4RegClassID;
2467 return RISCV::VRM8RegClassID;
2477 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2478 "Unexpected subreg numbering");
2479 return RISCV::sub_vrm1_0 +
Index;
2482 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2483 "Unexpected subreg numbering");
2484 return RISCV::sub_vrm2_0 +
Index;
2487 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2488 "Unexpected subreg numbering");
2489 return RISCV::sub_vrm4_0 +
Index;
2496 return RISCV::VRRegClassID;
2505std::pair<unsigned, unsigned>
2507 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2509 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2510 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2511 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2512 "Register classes not ordered");
2521 unsigned SubRegIdx = RISCV::NoSubRegister;
2522 for (
const unsigned RCID :
2523 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2524 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2528 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2533 return {SubRegIdx, InsertExtractIdx};
2538bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2567unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2574 "Unexpected opcode");
2576 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2578 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2581 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2655bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2656 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2665 "Expected legal fixed length vector!");
2668 unsigned MaxELen = Subtarget.
getELen();
2702 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2709 "Expected to convert into a scalable vector!");
2710 assert(V.getValueType().isFixedLengthVector() &&
2711 "Expected a fixed length vector operand!");
2721 "Expected to convert into a fixed length vector!");
2722 assert(V.getValueType().isScalableVector() &&
2723 "Expected a scalable vector operand!");
2751 const auto [MinVLMAX, MaxVLMAX] =
2753 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2759static std::pair<SDValue, SDValue>
2768static std::pair<SDValue, SDValue>
2781static std::pair<SDValue, SDValue>
2798std::pair<unsigned, unsigned>
2814 return std::make_pair(MinVLMAX, MaxVLMAX);
2826 EVT VT,
unsigned DefinedValues)
const {
2840 std::tie(LMul, Fractional) =
2843 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2845 Cost = (LMul * DLenFactor);
2890 MVT DstVT =
Op.getSimpleValueType();
2891 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2899 Src.getValueType() == MVT::bf16) {
2906 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2914 Opc,
DL, DstVT, Src,
2928 MVT SrcVT = Src.getSimpleValueType();
2934 if (SatVT != DstEltVT)
2938 if (SrcEltSize > (2 * DstEltSize))
2941 MVT DstContainerVT = DstVT;
2942 MVT SrcContainerVT = SrcVT;
2948 "Expected same element count");
2957 {Src, Src, DAG.getCondCode(ISD::SETNE),
2958 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2962 if (DstEltSize > (2 * SrcEltSize)) {
2976 Res, DAG.
getUNDEF(DstContainerVT), VL);
2988 case ISD::VP_FROUNDEVEN:
2992 case ISD::VP_FROUNDTOZERO:
2996 case ISD::VP_FFLOOR:
3004 case ISD::VP_FROUND:
3020 MVT VT =
Op.getSimpleValueType();
3027 MVT ContainerVT = VT;
3034 if (
Op->isVPOpcode()) {
3035 Mask =
Op.getOperand(1);
3039 VL =
Op.getOperand(2);
3061 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3075 switch (
Op.getOpcode()) {
3081 case ISD::VP_FFLOOR:
3084 case ISD::VP_FROUND:
3085 case ISD::VP_FROUNDEVEN:
3086 case ISD::VP_FROUNDTOZERO: {
3102 case ISD::VP_FNEARBYINT:
3115 Src, Src, Mask, VL);
3130 MVT VT =
Op.getSimpleValueType();
3134 MVT ContainerVT = VT;
3146 MVT MaskVT = Mask.getSimpleValueType();
3149 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3150 DAG.getUNDEF(MaskVT), Mask, VL});
3154 {Chain, Src, Src, Src, Unorder, VL});
3155 Chain = Src.getValue(1);
3171 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3183 switch (
Op.getOpcode()) {
3194 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3200 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3204 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3213 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3214 Truncated, Mask, VL);
3220 Src, Src, Mask, VL);
3230 MVT VT =
Op.getSimpleValueType();
3258 MVT VT =
Op.getSimpleValueType();
3263 MVT ContainerVT = VT;
3285 if (
Merge.isUndef())
3297 if (
Merge.isUndef())
3306 "Unexpected vector MVT");
3334 return std::nullopt;
3349 unsigned EltSizeInBits) {
3352 return std::nullopt;
3353 bool IsInteger =
Op.getValueType().isInteger();
3355 std::optional<unsigned> SeqStepDenom;
3356 std::optional<int64_t> SeqStepNum, SeqAddend;
3357 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3358 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3363 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3365 if (Elt.isUndef()) {
3366 Elts[
Idx] = std::nullopt;
3370 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3375 return std::nullopt;
3376 Elts[
Idx] = *ExactInteger;
3389 unsigned IdxDiff =
Idx - PrevElt->second;
3390 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3398 int64_t Remainder = ValDiff % IdxDiff;
3400 if (Remainder != ValDiff) {
3403 return std::nullopt;
3409 SeqStepNum = ValDiff;
3410 else if (ValDiff != SeqStepNum)
3411 return std::nullopt;
3414 SeqStepDenom = IdxDiff;
3415 else if (IdxDiff != *SeqStepDenom)
3416 return std::nullopt;
3420 if (!PrevElt || PrevElt->first != *Elt)
3421 PrevElt = std::make_pair(*Elt,
Idx);
3425 if (!SeqStepNum || !SeqStepDenom)
3426 return std::nullopt;
3434 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3435 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3438 else if (Addend != SeqAddend)
3439 return std::nullopt;
3442 assert(SeqAddend &&
"Must have an addend if we have a step");
3444 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3465 MVT ContainerVT = VT;
3493 MVT VT =
Op.getSimpleValueType();
3505 unsigned MostCommonCount = 0;
3507 unsigned NumUndefElts =
3515 unsigned NumScalarLoads = 0;
3521 ValueCounts.
insert(std::make_pair(V, 0));
3522 unsigned &Count = ValueCounts[V];
3524 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3525 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3530 if (++Count >= MostCommonCount) {
3532 MostCommonCount = Count;
3536 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3537 unsigned NumDefElts = NumElts - NumUndefElts;
3538 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3544 ((MostCommonCount > DominantValueCountThreshold) ||
3557 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3558 LastOp != DominantValue) {
3567 Processed.insert(LastOp);
3572 const SDValue &V = OpIdx.value();
3573 if (V.isUndef() || !Processed.insert(V).second)
3575 if (ValueCounts[V] == 1) {
3584 return DAG.getConstant(V == V1, DL, XLenVT);
3600 MVT VT =
Op.getSimpleValueType();
3630 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3631 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3639 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3640 MVT IntegerViaVecVT =
3645 unsigned BitPos = 0, IntegerEltIdx = 0;
3648 for (
unsigned I = 0;
I < NumElts;) {
3650 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3651 Bits |= ((
uint64_t)BitValue << BitPos);
3657 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3658 if (NumViaIntegerBits <= 32)
3659 Bits = SignExtend64<32>(Bits);
3661 Elts[IntegerEltIdx] = Elt;
3670 if (NumElts < NumViaIntegerBits) {
3674 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3702 int64_t StepNumerator = SimpleVID->StepNumerator;
3703 unsigned StepDenominator = SimpleVID->StepDenominator;
3704 int64_t Addend = SimpleVID->Addend;
3706 assert(StepNumerator != 0 &&
"Invalid step");
3707 bool Negate =
false;
3708 int64_t SplatStepVal = StepNumerator;
3712 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3714 Negate = StepNumerator < 0;
3716 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3723 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3724 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3726 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3729 MVT VIDContainerVT =
3737 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3738 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3740 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3742 if (StepDenominator != 1) {
3747 if (Addend != 0 || Negate) {
3766 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3767 "Unexpected sequence type");
3771 unsigned ViaVecLen =
3775 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3778 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3779 const auto &SeqV = OpIdx.value();
3780 if (!SeqV.isUndef())
3782 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3787 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3788 SplatValue = SignExtend64<32>(SplatValue);
3810 const auto *BV = cast<BuildVectorSDNode>(
Op);
3813 BV->getRepeatedSequence(Sequence) &&
3814 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3815 unsigned SeqLen = Sequence.size();
3817 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3818 ViaIntVT == MVT::i64) &&
3819 "Unexpected sequence type");
3824 const unsigned RequiredVL = NumElts / SeqLen;
3825 const unsigned ViaVecLen =
3827 NumElts : RequiredVL;
3830 unsigned EltIdx = 0;
3831 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3835 for (
const auto &SeqV : Sequence) {
3836 if (!SeqV.isUndef())
3838 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3844 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3845 SplatValue = SignExtend64<32>(SplatValue);
3851 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3852 "Unexpected bitcast sequence");
3853 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3856 MVT ViaContainerVT =
3863 if (ViaVecLen != RequiredVL)
3880 if (EltBitSize - SignBits < 8) {
3884 Source, DAG, Subtarget);
3901 MVT VT =
Op.getSimpleValueType();
3973 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3977 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3993 unsigned NumUndefElts =
3995 unsigned NumDefElts = NumElts - NumUndefElts;
3996 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4003 for (
unsigned i = 0; i < NumElts; i++) {
4005 if (i < NumElts / 2) {
4012 bool SelectMaskVal = (i < NumElts / 2);
4015 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4016 MaskVals.
size() == NumElts);
4051 unsigned UndefCount = 0;
4058 LinearBudget -= PerSlideCost;
4061 LinearBudget -= PerSlideCost;
4064 LinearBudget -= PerSlideCost;
4067 if (LinearBudget < 0)
4072 "Illegal type which will result in reserved encoding");
4097 Vec,
Offset, Mask, VL, Policy);
4110 Vec,
Offset, Mask, VL, Policy);
4120 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4121 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4122 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4125 if ((LoC >> 31) == HiC)
4136 (isa<RegisterSDNode>(VL) &&
4137 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4139 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4154 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4155 Hi.getConstantOperandVal(1) == 31)
4174 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4186 bool HasPassthru = Passthru && !Passthru.
isUndef();
4187 if (!HasPassthru && !Passthru)
4195 if (Scalar.getValueType().bitsLE(XLenVT)) {
4202 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4206 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4207 "Unexpected scalar for splat lowering!");
4231 SDValue ExtractedVal = Scalar.getOperand(0);
4236 MVT ExtractedContainerVT = ExtractedVT;
4239 DAG, ExtractedContainerVT, Subtarget);
4241 ExtractedVal, DAG, Subtarget);
4243 if (ExtractedContainerVT.
bitsLE(VT))
4258 if (!Scalar.getValueType().bitsLE(XLenVT))
4261 VT,
DL, DAG, Subtarget);
4269 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4295 if (Src != V2.getOperand(0))
4299 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4304 V2.getConstantOperandVal(1) != Mask.size())
4308 if (Mask[0] != 0 && Mask[0] != 1)
4313 for (
unsigned i = 1; i != Mask.size(); ++i)
4314 if (Mask[i] != Mask[i - 1] + 2)
4332 int Size = Mask.size();
4334 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4340 EvenSrc = StartIndexes[0];
4341 OddSrc = StartIndexes[1];
4344 if (EvenSrc != 0 && OddSrc != 0)
4354 int HalfNumElts = NumElts / 2;
4355 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4371 int Size = Mask.size();
4383 for (
int i = 0; i !=
Size; ++i) {
4389 int StartIdx = i - (M %
Size);
4397 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4400 Rotation = CandidateRotation;
4401 else if (Rotation != CandidateRotation)
4406 int MaskSrc = M <
Size ? 0 : 1;
4411 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4416 TargetSrc = MaskSrc;
4417 else if (TargetSrc != MaskSrc)
4424 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4425 assert((LoSrc >= 0 || HiSrc >= 0) &&
4426 "Failed to find a rotated input vector!");
4441 MVT ContainerVT = VT;
4444 assert(Src.getSimpleValueType().isFixedLengthVector());
4448 MVT SrcContainerVT =
4461 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4468 unsigned Shift = EvenElts ? 0 : EltBits;
4474 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4500 auto findNonEXTRACT_SUBVECTORParent =
4501 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4506 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4507 Offset += Parent.getConstantOperandVal(1);
4508 Parent = Parent.getOperand(0);
4510 return std::make_pair(Parent,
Offset);
4513 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4514 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4523 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4524 if (NewMask[i] == -1)
4527 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4528 NewMask[i] = NewMask[i] + V1IndexOffset;
4532 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4538 if (NewMask[0] <= 0)
4542 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4543 if (NewMask[i - 1] + 1 != NewMask[i])
4547 MVT SrcVT = Src.getSimpleValueType();
4578 int NumSubElts,
Index;
4583 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4584 SDValue InPlace = OpsSwapped ? V2 : V1;
4585 SDValue ToInsert = OpsSwapped ? V1 : V2;
4595 if (NumSubElts +
Index >= (
int)NumElts)
4609 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4621 bool OpsSwapped =
false;
4622 if (!isa<BuildVectorSDNode>(V1)) {
4623 if (!isa<BuildVectorSDNode>(V2))
4628 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4636 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4637 for (
unsigned i = S; i != E; ++i)
4638 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4644 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4645 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4648 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4650 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4655 auto OpCode = IsVSlidedown ?
4660 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4663 Splat, TrueMask, VL);
4674 MVT VecContainerVT = VecVT;
4691 MVT WideContainerVT = WideVT;
4697 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4710 }
else if (Subtarget.hasStdExtZvbb()) {
4715 OffsetVec, Passthru, Mask, VL);
4718 Interleaved, EvenV, Passthru, Mask, VL);
4726 Interleaved, OffsetVec, Passthru, Mask, VL);
4734 OddV, Passthru, Mask, VL);
4740 OddV, AllOnesVec, Passthru, Mask, VL);
4748 Interleaved, OddsMul, Passthru, Mask, VL);
4755 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4801 if (ViaEltSize > NumElts)
4810 if (ViaEltSize > NumElts)
4816 if (ViaEltSize > NumElts)
4825 MVT &RotateVT,
unsigned &RotateAmt) {
4831 unsigned NumSubElts;
4833 NumElts, NumSubElts, RotateAmt))
4836 NumElts / NumSubElts);
4898 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4901 OutMasks(VRegsPerSrc, {-1, {}});
4906 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4907 int DstVecIdx = DstIdx / ElemsPerVReg;
4908 int DstSubIdx = DstIdx % ElemsPerVReg;
4909 int SrcIdx = Mask[DstIdx];
4910 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4912 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4913 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4914 if (OutMasks[DstVecIdx].first == -1)
4915 OutMasks[DstVecIdx].first = SrcVecIdx;
4916 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4922 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4923 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4937 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4938 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4939 if (SrcVecIdx == -1)
4941 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4948 unsigned InsertIdx = DstVecIdx * NumOpElts;
4961 MVT VT =
Op.getSimpleValueType();
4976 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5000 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5001 V = V.getOperand(
Offset / OpElements);
5007 auto *Ld = cast<LoadSDNode>(V);
5017 SDValue Ops[] = {Ld->getChain(),
5035 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5036 Ld->getPointerInfo().getWithOffset(
Offset),
5037 Ld->getOriginalAlign(),
5041 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5042 Ld->getOriginalAlign(),
5043 Ld->getMemOperand()->getFlags());
5054 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5057 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5079 if (Subtarget.hasStdExtZvkb())
5090 LoV = LoSrc == 0 ? V1 : V2;
5094 HiV = HiSrc == 0 ? V1 : V2;
5100 unsigned InvRotate = NumElts - Rotation;
5110 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5130 int EvenSrc, OddSrc;
5135 int Size = Mask.size();
5137 assert(EvenSrc >= 0 &&
"Undef source?");
5138 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5142 assert(OddSrc >= 0 &&
"Undef source?");
5143 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5152 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5161 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5190 MVT IndexContainerVT =
5195 for (
int MaskIndex : Mask) {
5196 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5205 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5219 int MaskIndex = MaskIdx.value();
5220 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
5225 for (
int MaskIndex : Mask) {
5226 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
5233 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5246 for (
int MaskIndex : Mask) {
5247 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5249 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5250 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5252 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5257 std::swap(ShuffleMaskLHS, ShuffleMaskRHS);
5260 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5296RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5298 MVT VT =
Op.getSimpleValueType();
5302 MVT ContainerVT = VT;
5305 if (
Op->isVPOpcode()) {
5306 Mask =
Op.getOperand(1);
5310 VL =
Op.getOperand(2);
5316 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5318 FloatEltVT = MVT::f32;
5325 "Expected legal float type!");
5332 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5335 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5340 if (FloatVT.
bitsGT(VT)) {
5341 if (
Op->isVPOpcode())
5342 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5351 if (!
Op->isVPOpcode())
5355 MVT ContainerFloatVT =
5358 Src, Mask, RTZRM, VL);
5365 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5369 if (
Op->isVPOpcode()) {
5378 else if (IntVT.
bitsGT(VT))
5383 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5388 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5389 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5394 unsigned Adjust = ExponentBias + (EltSize - 1);
5396 if (
Op->isVPOpcode())
5406 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5407 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5417 MVT SrcVT =
Source.getSimpleValueType();
5426 SrcVT = ContainerVT;
5439 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5456 auto *
Load = cast<LoadSDNode>(
Op);
5457 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5460 Load->getMemoryVT(),
5461 *
Load->getMemOperand()))
5465 MVT VT =
Op.getSimpleValueType();
5467 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5468 "Unexpected unaligned RVV load type");
5472 "Expecting equally-sized RVV vector types to be legal");
5474 Load->getPointerInfo(),
Load->getOriginalAlign(),
5475 Load->getMemOperand()->getFlags());
5485 auto *
Store = cast<StoreSDNode>(
Op);
5486 assert(Store &&
Store->getValue().getValueType().isVector() &&
5487 "Expected vector store");
5490 Store->getMemoryVT(),
5491 *
Store->getMemOperand()))
5498 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5499 "Unexpected unaligned RVV store type");
5503 "Expecting equally-sized RVV vector types to be legal");
5504 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5506 Store->getPointerInfo(),
Store->getOriginalAlign(),
5507 Store->getMemOperand()->getFlags());
5512 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5514 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5541 unsigned ShiftAmt, AddOpc;
5558 if (Subtarget.hasStdExtZtso()) {
5581 "Unexpected custom legalisation");
5602 "Unexpected custom legalisation");
5617 "Unexpected custom legalisation");
5618 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5638 "Unexpected custom legalisation");
5654 MVT VT =
Op.getSimpleValueType();
5656 unsigned Check =
Op.getConstantOperandVal(1);
5657 unsigned TDCMask = 0;
5685 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5690 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5692 VL =
Op.getOperand(3);
5695 VL,
Op->getFlags());
5710 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5712 MVT MaskContainerVT =
5715 VL =
Op.getOperand(3);
5720 Mask, VL,
Op->getFlags());
5723 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5728 DAG.
getUNDEF(ContainerVT), Mask, VL});
5732 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5736 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5740 DAG.
getUNDEF(ContainerVT), Mask, VL});
5756 MVT VT =
Op.getSimpleValueType();
5783 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5790 MVT ContainerVT = VT;
5798 if (
Op->isVPOpcode()) {
5799 Mask =
Op.getOperand(2);
5803 VL =
Op.getOperand(3);
5811 {X, X, DAG.getCondCode(ISD::SETOEQ),
5812 DAG.getUNDEF(ContainerVT), Mask, VL});
5820 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5821 DAG.getUNDEF(ContainerVT), Mask, VL});
5831 DAG.
getUNDEF(ContainerVT), Mask, VL);
5839#define OP_CASE(NODE) \
5841 return RISCVISD::NODE##_VL;
5842#define VP_CASE(NODE) \
5843 case ISD::VP_##NODE: \
5844 return RISCVISD::NODE##_VL;
5846 switch (
Op.getOpcode()) {
5924 case ISD::VP_CTLZ_ZERO_UNDEF:
5927 case ISD::VP_CTTZ_ZERO_UNDEF:
5936 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5941 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5946 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5949 case ISD::VP_SELECT:
5958 case ISD::VP_SIGN_EXTEND:
5960 case ISD::VP_ZERO_EXTEND:
5962 case ISD::VP_FP_TO_SINT:
5964 case ISD::VP_FP_TO_UINT:
5967 case ISD::VP_FMINNUM:
5970 case ISD::VP_FMAXNUM:
5975 case ISD::VP_LLRINT:
5987 "not a RISC-V target specific op");
5993 "adding target specific op should update this function");
6013 "not a RISC-V target specific op");
6019 "adding target specific op should update this function");
6038 if (!
Op.getOperand(j).getValueType().isVector()) {
6039 LoOperands[j] =
Op.getOperand(j);
6040 HiOperands[j] =
Op.getOperand(j);
6043 std::tie(LoOperands[j], HiOperands[j]) =
6048 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6050 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6065 std::tie(LoOperands[j], HiOperands[j]) =
6069 if (!
Op.getOperand(j).getValueType().isVector()) {
6070 LoOperands[j] =
Op.getOperand(j);
6071 HiOperands[j] =
Op.getOperand(j);
6074 std::tie(LoOperands[j], HiOperands[j]) =
6079 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6081 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6091 auto [EVLLo, EVLHi] =
6092 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6096 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6098 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6116 if (!
Op.getOperand(j).getValueType().isVector()) {
6117 LoOperands[j] =
Op.getOperand(j);
6118 HiOperands[j] =
Op.getOperand(j);
6121 std::tie(LoOperands[j], HiOperands[j]) =
6126 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6129 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6138 switch (
Op.getOpcode()) {
6144 return lowerGlobalAddress(
Op, DAG);
6146 return lowerBlockAddress(
Op, DAG);
6148 return lowerConstantPool(
Op, DAG);
6150 return lowerJumpTable(
Op, DAG);
6152 return lowerGlobalTLSAddress(
Op, DAG);
6156 return lowerSELECT(
Op, DAG);
6158 return lowerBRCOND(
Op, DAG);
6160 return lowerVASTART(
Op, DAG);
6162 return lowerFRAMEADDR(
Op, DAG);
6164 return lowerRETURNADDR(
Op, DAG);
6171 return lowerShiftLeftParts(
Op, DAG);
6173 return lowerShiftRightParts(
Op, DAG,
true);
6175 return lowerShiftRightParts(
Op, DAG,
false);
6178 if (
Op.getValueType().isFixedLengthVector()) {
6179 assert(Subtarget.hasStdExtZvkb());
6180 return lowerToScalableOp(
Op, DAG);
6182 assert(Subtarget.hasVendorXTHeadBb() &&
6183 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6184 "Unexpected custom legalization");
6186 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6191 EVT VT =
Op.getValueType();
6195 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6201 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6202 Subtarget.hasStdExtZfbfmin()) {
6207 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6214 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6231 "Unexpected types");
6265 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6267 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6269 return LowerINTRINSIC_VOID(
Op, DAG);
6271 return LowerIS_FPCLASS(
Op, DAG);
6273 MVT VT =
Op.getSimpleValueType();
6275 assert(Subtarget.hasStdExtZvbb());
6276 return lowerToScalableOp(
Op, DAG);
6279 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6287 if (!
Op.getSimpleValueType().isVector())
6289 return lowerVectorTruncLike(
Op, DAG);
6292 if (
Op.getOperand(0).getValueType().isVector() &&
6293 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6294 return lowerVectorMaskExt(
Op, DAG, 1);
6297 if (
Op.getOperand(0).getValueType().isVector() &&
6298 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6299 return lowerVectorMaskExt(
Op, DAG, -1);
6302 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6304 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6306 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6308 MVT VT =
Op.getSimpleValueType();
6316 MVT ContainerVT = VT;
6322 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6329 MVT VT =
Op.getSimpleValueType();
6349 }
else if ((Val % 8) == 0) {
6365 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6366 Op.getOperand(1).getValueType() == MVT::i32) {
6378 if (
Op.getValueType() == MVT::nxv32f16 &&
6385 EVT VT =
Op.getValueType();
6388 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6390 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6396 if (!
Op.getValueType().isVector())
6398 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6402 EVT VT =
Op.getValueType();
6405 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6407 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6415 if (!
Op.getValueType().isVector())
6417 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6421 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6424 if (
Op.getValueType().isVector() &&
6425 Op.getValueType().getScalarType() == MVT::f16 &&
6428 if (
Op.getValueType() == MVT::nxv32f16)
6443 Op1.getValueType().isVector() &&
6444 Op1.getValueType().getScalarType() == MVT::f16 &&
6447 if (Op1.getValueType() == MVT::nxv32f16)
6452 Op1.getValueType().getVectorElementCount());
6455 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6465 MVT VT =
Op.getSimpleValueType();
6469 bool IsStrict =
Op->isStrictFPOpcode();
6470 SDValue Src =
Op.getOperand(0 + IsStrict);
6472 MVT SrcVT = Src.getSimpleValueType();
6477 "Unexpected vector element types");
6481 if (EltSize > (2 * SrcEltSize)) {
6493 Op.getOperand(0), Ext);
6497 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6502 auto [FExt, Chain] =
6504 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6511 if (SrcEltSize > (2 * EltSize)) {
6514 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6519 Op.getOperand(0), Src);
6534 Op.getOperand(0), Src);
6548 unsigned RVVOpc = 0;
6549 switch (
Op.getOpcode()) {
6581 "Expected same element count");
6588 Op.getOperand(0), Src, Mask, VL);
6592 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6607 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6614 MVT VT =
Op.getSimpleValueType();
6636 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6652 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6672 return lowerVECREDUCE(
Op, DAG);
6676 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6677 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6678 return lowerVECREDUCE(
Op, DAG);
6685 return lowerFPVECREDUCE(
Op, DAG);
6686 case ISD::VP_REDUCE_ADD:
6687 case ISD::VP_REDUCE_UMAX:
6688 case ISD::VP_REDUCE_SMAX:
6689 case ISD::VP_REDUCE_UMIN:
6690 case ISD::VP_REDUCE_SMIN:
6691 case ISD::VP_REDUCE_FADD:
6692 case ISD::VP_REDUCE_SEQ_FADD:
6693 case ISD::VP_REDUCE_FMIN:
6694 case ISD::VP_REDUCE_FMAX:
6695 case ISD::VP_REDUCE_FMINIMUM:
6696 case ISD::VP_REDUCE_FMAXIMUM:
6697 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6701 return lowerVPREDUCE(
Op, DAG);
6702 case ISD::VP_REDUCE_AND:
6703 case ISD::VP_REDUCE_OR:
6704 case ISD::VP_REDUCE_XOR:
6705 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6706 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6707 return lowerVPREDUCE(
Op, DAG);
6708 case ISD::VP_CTTZ_ELTS:
6709 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6710 return lowerVPCttzElements(
Op, DAG);
6714 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6717 return lowerINSERT_SUBVECTOR(
Op, DAG);
6719 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6721 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6723 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6725 return lowerSTEP_VECTOR(
Op, DAG);
6727 return lowerVECTOR_REVERSE(
Op, DAG);
6729 return lowerVECTOR_SPLICE(
Op, DAG);
6733 if ((
Op.getValueType().getScalarType() == MVT::f16 &&
6737 (
Op.getValueType().getScalarType() == MVT::bf16 &&
6739 if (
Op.getValueType() == MVT::nxv32f16 ||
6740 Op.getValueType() == MVT::nxv32bf16)
6752 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6753 return lowerVectorMaskSplat(
Op, DAG);
6761 MVT VT =
Op.getSimpleValueType();
6762 MVT ContainerVT = VT;
6780 Op->ops().take_front(HalfNumOps));
6782 Op->ops().drop_front(HalfNumOps));
6786 unsigned NumOpElts =
6787 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6790 SDValue SubVec = OpIdx.value();
6801 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6803 if (
Op.getValueType().isFixedLengthVector())
6804 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6807 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6809 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6810 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6814 return lowerMaskedLoad(
Op, DAG);
6817 return lowerMaskedStore(
Op, DAG);
6826 EVT VT =
Op.getValueType();
6837 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6839 MVT VT =
Op.getSimpleValueType();
6844 "Unexpected CondCode");
6852 if (isa<ConstantSDNode>(
RHS)) {
6853 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6854 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6873 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6878 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6894 return lowerToScalableOp(
Op, DAG);
6898 if (
Op.getSimpleValueType().isFixedLengthVector())
6899 return lowerToScalableOp(
Op, DAG);
6901 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6902 "Unexpected custom legalisation");
6914 if (
Op.getValueType() == MVT::nxv32f16 &&
6927 return lowerToScalableOp(
Op, DAG);
6930 if (!
Op.getValueType().isVector())
6932 return lowerToScalableOp(
Op, DAG);
6935 if (!
Op.getValueType().isVector())
6937 return lowerToScalableOp(
Op, DAG);
6941 EVT VT =
Op->getValueType(0);
6956 return lowerABS(
Op, DAG);
6961 if (Subtarget.hasStdExtZvbb())
6962 return lowerToScalableOp(
Op, DAG);
6964 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6966 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6968 if (
Op.getValueType() == MVT::nxv32f16 &&
6972 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6979 if (
Op.getValueType() == MVT::nxv32f16 &&
6983 return lowerToScalableOp(
Op, DAG);
6986 return lowerVectorStrictFSetcc(
Op, DAG);
6996 case ISD::VP_GATHER:
6997 return lowerMaskedGather(
Op, DAG);
6999 case ISD::VP_SCATTER:
7000 return lowerMaskedScatter(
Op, DAG);
7002 return lowerGET_ROUNDING(
Op, DAG);
7004 return lowerSET_ROUNDING(
Op, DAG);
7006 return lowerEH_DWARF_CFA(
Op, DAG);
7007 case ISD::VP_SELECT:
7016 case ISD::VP_UADDSAT:
7017 case ISD::VP_USUBSAT:
7018 case ISD::VP_SADDSAT:
7019 case ISD::VP_SSUBSAT:
7021 case ISD::VP_LLRINT:
7022 return lowerVPOp(
Op, DAG);
7026 return lowerLogicVPOp(
Op, DAG);
7035 case ISD::VP_FMINNUM:
7036 case ISD::VP_FMAXNUM:
7037 case ISD::VP_FCOPYSIGN:
7038 if (
Op.getValueType() == MVT::nxv32f16 &&
7046 return lowerVPOp(
Op, DAG);
7047 case ISD::VP_IS_FPCLASS:
7048 return LowerIS_FPCLASS(
Op, DAG);
7049 case ISD::VP_SIGN_EXTEND:
7050 case ISD::VP_ZERO_EXTEND:
7051 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7052 return lowerVPExtMaskOp(
Op, DAG);
7053 return lowerVPOp(
Op, DAG);
7054 case ISD::VP_TRUNCATE:
7055 return lowerVectorTruncLike(
Op, DAG);
7056 case ISD::VP_FP_EXTEND:
7057 case ISD::VP_FP_ROUND:
7058 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7059 case ISD::VP_SINT_TO_FP:
7060 case ISD::VP_UINT_TO_FP:
7061 if (
Op.getValueType().isVector() &&
7062 Op.getValueType().getScalarType() == MVT::f16 &&
7065 if (
Op.getValueType() == MVT::nxv32f16)
7077 case ISD::VP_FP_TO_SINT:
7078 case ISD::VP_FP_TO_UINT:
7080 Op1.getValueType().isVector() &&
7081 Op1.getValueType().getScalarType() == MVT::f16 &&
7084 if (Op1.getValueType() == MVT::nxv32f16)
7089 Op1.getValueType().getVectorElementCount());
7093 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7095 return lowerVPFPIntConvOp(
Op, DAG);
7097 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
7101 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7102 return lowerVPSetCCMaskOp(
Op, DAG);
7108 case ISD::VP_BITREVERSE:
7110 return lowerVPOp(
Op, DAG);
7112 case ISD::VP_CTLZ_ZERO_UNDEF:
7113 if (Subtarget.hasStdExtZvbb())
7114 return lowerVPOp(
Op, DAG);
7115 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7117 case ISD::VP_CTTZ_ZERO_UNDEF:
7118 if (Subtarget.hasStdExtZvbb())
7119 return lowerVPOp(
Op, DAG);
7120 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7122 return lowerVPOp(
Op, DAG);
7123 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7124 return lowerVPStridedLoad(
Op, DAG);
7125 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7126 return lowerVPStridedStore(
Op, DAG);
7128 case ISD::VP_FFLOOR:
7130 case ISD::VP_FNEARBYINT:
7131 case ISD::VP_FROUND:
7132 case ISD::VP_FROUNDEVEN:
7133 case ISD::VP_FROUNDTOZERO:
7134 if (
Op.getValueType() == MVT::nxv32f16 &&
7139 case ISD::VP_FMAXIMUM:
7140 case ISD::VP_FMINIMUM:
7141 if (
Op.getValueType() == MVT::nxv32f16 &&
7146 case ISD::EXPERIMENTAL_VP_SPLICE:
7147 return lowerVPSpliceExperimental(
Op, DAG);
7148 case ISD::EXPERIMENTAL_VP_REVERSE:
7149 return lowerVPReverseExperimental(
Op, DAG);
7152 "llvm.clear_cache only needs custom lower on Linux targets");
7155 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7156 Op.getOperand(2), Flags,
DL);
7164 MakeLibCallOptions CallOptions;
7165 std::pair<SDValue, SDValue> CallResult =
7166 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7167 {Start,
End, Flags}, CallOptions,
DL, InChain);
7170 return CallResult.second;
7187 N->getOffset(), Flags);
7195template <
class NodeTy>
7197 bool IsLocal,
bool IsExternWeak)
const {
7207 if (IsLocal && !Subtarget.allowTaggedGlobals())
7269 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7278 return getAddr(
N, DAG);
7285 return getAddr(
N, DAG);
7292 return getAddr(
N, DAG);
7297 bool UseGOT)
const {
7361 Args.push_back(Entry);
7394 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7408 Addr = getStaticTLSAddr(
N, DAG,
false);
7411 Addr = getStaticTLSAddr(
N, DAG,
true);
7416 : getDynamicTLSAddr(
N, DAG);
7433 if (
LHS == LHS2 &&
RHS == RHS2) {
7438 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7446 return std::nullopt;
7454 MVT VT =
N->getSimpleValueType(0);
7484 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7487 if (~TrueVal == FalseVal) {
7527 if (Subtarget.hasShortForwardBranchOpt())
7530 unsigned SelOpNo = 0;
7540 unsigned ConstSelOpNo = 1;
7541 unsigned OtherSelOpNo = 2;
7542 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7547 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7548 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7552 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7553 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7559 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7561 std::swap(NewConstOps[0], NewConstOps[1]);
7573 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7575 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7578 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7579 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7588 MVT VT =
Op.getSimpleValueType();
7602 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7630 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7634 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7636 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7637 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7639 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7644 DL, VT, LHSVal, CondV);
7660 if (
Op.hasOneUse()) {
7661 unsigned UseOpc =
Op->use_begin()->getOpcode();
7667 return lowerSELECT(NewSel, DAG);
7695 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7716 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7720 if (TrueVal - 1 == FalseVal)
7722 if (TrueVal + 1 == FalseVal)
7729 RHS == TrueV && LHS == FalseV) {
7746 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7772 LHS, RHS, TargetCC,
Op.getOperand(2));
7790 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7802 int XLenInBytes = Subtarget.
getXLen() / 8;
7804 EVT VT =
Op.getValueType();
7807 unsigned Depth =
Op.getConstantOperandVal(0);
7809 int Offset = -(XLenInBytes * 2);
7825 int XLenInBytes = Subtarget.
getXLen() / 8;
7830 EVT VT =
Op.getValueType();
7832 unsigned Depth =
Op.getConstantOperandVal(0);
7834 int Off = -XLenInBytes;
7835 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7854 EVT VT =
Lo.getValueType();
7893 EVT VT =
Lo.getValueType();
7944 MVT VT =
Op.getSimpleValueType();
7970 MVT VecVT =
Op.getSimpleValueType();
7972 "Unexpected SPLAT_VECTOR_PARTS lowering");
7978 MVT ContainerVT = VecVT;
7998 int64_t ExtTrueVal)
const {
8000 MVT VecVT =
Op.getSimpleValueType();
8003 assert(Src.getValueType().isVector() &&
8004 Src.getValueType().getVectorElementType() == MVT::i1);
8025 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8027 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
8030 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
8035SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
8037 MVT ExtVT =
Op.getSimpleValueType();
8041 MVT VT =
Op.getOperand(0).getSimpleValueType();
8067 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8069 EVT MaskVT =
Op.getValueType();
8072 "Unexpected type for vector mask lowering");
8074 MVT VecVT = Src.getSimpleValueType();
8078 VL =
Op.getOperand(2);
8081 MVT ContainerVT = VecVT;
8087 MVT MaskContainerVT =
8094 std::tie(Mask, VL) =
8102 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8104 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8108 DAG.
getUNDEF(ContainerVT), Mask, VL);
8111 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8119 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8122 MVT VT =
Op.getSimpleValueType();
8124 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8128 return lowerVectorMaskTruncLike(
Op, DAG);
8136 MVT SrcVT = Src.getSimpleValueType();
8141 "Unexpected vector truncate lowering");
8143 MVT ContainerVT = SrcVT;
8147 VL =
Op.getOperand(2);
8160 std::tie(Mask, VL) =
8171 }
while (SrcEltVT != DstEltVT);
8180RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8185 MVT VT =
Op.getSimpleValueType();
8186 MVT SrcVT = Src.getSimpleValueType();
8187 MVT ContainerVT = VT;
8210 Chain, Src, Mask, VL);
8211 Chain = Src.getValue(1);
8218 Chain, Src, Mask, VL);
8229RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8232 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8239 MVT VT =
Op.getSimpleValueType();
8241 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8244 MVT SrcVT = Src.getSimpleValueType();
8246 bool IsDirectExtend =
8254 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8257 MVT ContainerVT = VT;
8261 VL =
Op.getOperand(2);
8275 std::tie(Mask, VL) =
8281 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8287 unsigned InterConvOpc =
8292 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8294 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8305static std::optional<MVT>
8311 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8313 if (MaxIdx < MinVLMAX)
8315 else if (MaxIdx < MinVLMAX * 2)
8317 else if (MaxIdx < MinVLMAX * 4)
8322 return std::nullopt;
8335 MVT VecVT =
Op.getSimpleValueType();
8349 MVT ContainerVT = VecVT;
8358 MVT OrigContainerVT = ContainerVT;
8361 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8362 const unsigned OrigIdx = IdxC->getZExtValue();
8365 DL, DAG, Subtarget)) {
8366 ContainerVT = *ShrunkVT;
8375 VLEN && ContainerVT.
bitsGT(M1VT)) {
8378 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8379 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8380 unsigned ExtractIdx =
8399 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8400 const auto *CVal = cast<ConstantSDNode>(Val);
8401 if (isInt<32>(CVal->getSExtValue())) {
8402 IsLegalInsert =
true;
8411 if (IsLegalInsert) {
8417 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8433 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8434 MVT I32ContainerVT =
8445 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8450 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8452 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8457 ValInVec, AlignedIdx);
8467 DAG.
getUNDEF(I32ContainerVT), ValLo,
8468 I32Mask, InsertI64VL);
8470 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8471 I32Mask, InsertI64VL);
8473 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8486 Idx, Mask, InsertVL, Policy);
8490 Slideup, AlignedIdx);
8505 EVT EltVT =
Op.getValueType();
8512 MVT ContainerVT = VecVT;
8528 unsigned WidenVecLen;
8531 unsigned MaxEEW = Subtarget.
getELen();
8536 "the number of elements should be power of 2");
8540 ExtractBitIdx =
Idx;
8542 WideEltVT = LargestEltVT;
8545 ExtractElementIdx = DAG.
getNode(
8556 Vec, ExtractElementIdx);
8572 MVT ContainerVT = VecVT;
8583 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8584 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8586 unsigned OrigIdx = IdxC->getZExtValue();
8589 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8590 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8591 unsigned ExtractIdx =
8601 std::optional<uint64_t> MaxIdx;
8604 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8605 MaxIdx = IdxC->getZExtValue();
8607 if (
auto SmallerVT =
8609 ContainerVT = *SmallerVT;
8656 "Unexpected opcode");
8663 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8668 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8669 if (!
II || !
II->hasScalarOperand())
8672 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
8685 if (OpVT.
bitsLT(XLenVT)) {
8692 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8702 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
8703 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8706 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8717 case Intrinsic::riscv_vslide1up:
8718 case Intrinsic::riscv_vslide1down:
8719 case Intrinsic::riscv_vslide1up_mask:
8720 case Intrinsic::riscv_vslide1down_mask: {
8723 bool IsMasked = NumOps == 7;
8729 std::tie(ScalarLo, ScalarHi) =
8737 if (isa<ConstantSDNode>(AVL)) {
8738 const auto [MinVLMAX, MaxVLMAX] =
8742 if (AVLInt <= MinVLMAX) {
8744 }
else if (AVLInt >= 2 * MaxVLMAX) {
8751 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8785 if (IntNo == Intrinsic::riscv_vslide1up ||
8786 IntNo == Intrinsic::riscv_vslide1up_mask) {
8788 ScalarHi, I32Mask, I32VL);
8790 ScalarLo, I32Mask, I32VL);
8793 ScalarLo, I32Mask, I32VL);
8795 ScalarHi, I32Mask, I32VL);
8844 const unsigned ElementWidth = 8;
8849 [[maybe_unused]]
unsigned MinVF =
8852 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8856 bool Fractional = VF < LMul1VF;
8857 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8878 MVT ContainerVT = OpVT;
8905 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8909 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8910 if (!
II || !
II->hasScalarOperand())
8913 unsigned SplatOp =
II->ScalarOperand + 1;
8926 if (OpVT.
bitsLT(XLenVT)) {
8929 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8942 EVT ValType = V.getValueType();
8943 if (ValType.isVector() && ValType.isFloatingPoint()) {
8946 ValType.getVectorElementCount());
8949 if (ValType.isFixedLengthVector()) {
8951 DAG, V.getSimpleValueType(), Subtarget);
8967 unsigned IntNo =
Op.getConstantOperandVal(0);
8974 case Intrinsic::thread_pointer: {
8978 case Intrinsic::riscv_orc_b:
8979 case Intrinsic::riscv_brev8:
8980 case Intrinsic::riscv_sha256sig0:
8981 case Intrinsic::riscv_sha256sig1:
8982 case Intrinsic::riscv_sha256sum0:
8983 case Intrinsic::riscv_sha256sum1:
8984 case Intrinsic::riscv_sm3p0:
8985 case Intrinsic::riscv_sm3p1: {
9005 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9007 case Intrinsic::riscv_sm4ks:
9008 case Intrinsic::riscv_sm4ed: {
9018 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
9022 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
9025 case Intrinsic::riscv_zip:
9026 case Intrinsic::riscv_unzip: {
9029 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9031 case Intrinsic::riscv_mopr: {
9044 case Intrinsic::riscv_moprr: {
9056 Op.getOperand(2),
Op.getOperand(3));
9058 case Intrinsic::riscv_clmul:
9069 case Intrinsic::riscv_clmulh:
9070 case Intrinsic::riscv_clmulr: {
9088 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9090 case Intrinsic::experimental_get_vector_length:
9092 case Intrinsic::experimental_cttz_elts:
9094 case Intrinsic::riscv_vmv_x_s: {
9098 case Intrinsic::riscv_vfmv_f_s:
9101 case Intrinsic::riscv_vmv_v_x:
9103 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9105 case Intrinsic::riscv_vfmv_v_f:
9107 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9108 case Intrinsic::riscv_vmv_s_x: {
9111 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9114 Op.getOperand(1), Scalar,
Op.getOperand(3));
9117 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9134 MVT VT =
Op.getSimpleValueType();
9139 if (
Op.getOperand(1).isUndef())
9155 case Intrinsic::riscv_vfmv_s_f:
9157 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9159 case Intrinsic::riscv_vaesdf_vv:
9160 case Intrinsic::riscv_vaesdf_vs:
9161 case Intrinsic::riscv_vaesdm_vv:
9162 case Intrinsic::riscv_vaesdm_vs:
9163 case Intrinsic::riscv_vaesef_vv:
9164 case Intrinsic::riscv_vaesef_vs:
9165 case Intrinsic::riscv_vaesem_vv:
9166 case Intrinsic::riscv_vaesem_vs:
9167 case Intrinsic::riscv_vaeskf1:
9168 case Intrinsic::riscv_vaeskf2:
9169 case Intrinsic::riscv_vaesz_vs:
9170 case Intrinsic::riscv_vsm4k:
9171 case Intrinsic::riscv_vsm4r_vv:
9172 case Intrinsic::riscv_vsm4r_vs: {
9173 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9174 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9175 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9180 case Intrinsic::riscv_vsm3c:
9181 case Intrinsic::riscv_vsm3me: {
9182 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9183 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9188 case Intrinsic::riscv_vsha2ch:
9189 case Intrinsic::riscv_vsha2cl:
9190 case Intrinsic::riscv_vsha2ms: {
9191 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9192 !Subtarget.hasStdExtZvknhb())
9194 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9195 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9196 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9200 case Intrinsic::riscv_sf_vc_v_x:
9201 case Intrinsic::riscv_sf_vc_v_i:
9202 case Intrinsic::riscv_sf_vc_v_xv:
9203 case Intrinsic::riscv_sf_vc_v_iv:
9204 case Intrinsic::riscv_sf_vc_v_vv:
9205 case Intrinsic::riscv_sf_vc_v_fv:
9206 case Intrinsic::riscv_sf_vc_v_xvv:
9207 case Intrinsic::riscv_sf_vc_v_ivv:
9208 case Intrinsic::riscv_sf_vc_v_vvv:
9209 case Intrinsic::riscv_sf_vc_v_fvv:
9210 case Intrinsic::riscv_sf_vc_v_xvw:
9211 case Intrinsic::riscv_sf_vc_v_ivw:
9212 case Intrinsic::riscv_sf_vc_v_vvw:
9213 case Intrinsic::riscv_sf_vc_v_fvw: {
9214 MVT VT =
Op.getSimpleValueType();
9251 MVT VT =
Op.getSimpleValueType();
9255 if (VT.isFloatingPoint()) {
9260 if (VT.isFixedLengthVector())
9270 if (VT.isFixedLengthVector())
9272 if (VT.isFloatingPoint())
9291 unsigned IntNo =
Op.getConstantOperandVal(1);
9295 case Intrinsic::riscv_masked_strided_load: {
9304 MVT VT =
Op->getSimpleValueType(0);
9305 MVT ContainerVT = VT;
9318 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9330 ScalarVT,
Load->getMemOperand());
9336 Load->getMemOperand());
9341 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9346 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9348 Ops.push_back(PassThru);
9350 Ops.push_back(Stride);
9352 Ops.push_back(Mask);
9357 Ops.push_back(Policy);
9363 Load->getMemoryVT(),
Load->getMemOperand());
9364 Chain =
Result.getValue(1);
9370 case Intrinsic::riscv_seg2_load:
9371 case Intrinsic::riscv_seg3_load:
9372 case Intrinsic::riscv_seg4_load:
9373 case Intrinsic::riscv_seg5_load:
9374 case Intrinsic::riscv_seg6_load:
9375 case Intrinsic::riscv_seg7_load:
9376 case Intrinsic::riscv_seg8_load: {
9379 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9380 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9381 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9382 Intrinsic::riscv_vlseg8};
9383 unsigned NF =
Op->getNumValues() - 1;
9384 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9386 MVT VT =
Op->getSimpleValueType(0);
9392 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9394 ContainerVTs.push_back(MVT::Other);
9402 Load->getMemoryVT(),
Load->getMemOperand());
9404 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9410 case Intrinsic::riscv_sf_vc_v_x_se:
9412 case Intrinsic::riscv_sf_vc_v_i_se:
9414 case Intrinsic::riscv_sf_vc_v_xv_se:
9416 case Intrinsic::riscv_sf_vc_v_iv_se:
9418 case Intrinsic::riscv_sf_vc_v_vv_se:
9420 case Intrinsic::riscv_sf_vc_v_fv_se:
9422 case Intrinsic::riscv_sf_vc_v_xvv_se:
9424 case Intrinsic::riscv_sf_vc_v_ivv_se:
9426 case Intrinsic::riscv_sf_vc_v_vvv_se:
9428 case Intrinsic::riscv_sf_vc_v_fvv_se:
9430 case Intrinsic::riscv_sf_vc_v_xvw_se:
9432 case Intrinsic::riscv_sf_vc_v_ivw_se:
9434 case Intrinsic::riscv_sf_vc_v_vvw_se:
9436 case Intrinsic::riscv_sf_vc_v_fvw_se:
9445 unsigned IntNo =
Op.getConstantOperandVal(1);
9449 case Intrinsic::riscv_masked_strided_store: {
9460 MVT ContainerVT = VT;
9474 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9477 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9487 Ops,
Store->getMemoryVT(),
9488 Store->getMemOperand());
9490 case Intrinsic::riscv_seg2_store:
9491 case Intrinsic::riscv_seg3_store:
9492 case Intrinsic::riscv_seg4_store:
9493 case Intrinsic::riscv_seg5_store:
9494 case Intrinsic::riscv_seg6_store:
9495 case Intrinsic::riscv_seg7_store:
9496 case Intrinsic::riscv_seg8_store: {
9499 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9500 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9501 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9502 Intrinsic::riscv_vsseg8};
9505 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9507 MVT VT =
Op->getOperand(2).getSimpleValueType();
9515 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9517 for (
unsigned i = 0; i < NF; i++)
9519 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9524 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9526 case Intrinsic::riscv_sf_vc_xv_se:
9528 case Intrinsic::riscv_sf_vc_iv_se:
9530 case Intrinsic::riscv_sf_vc_vv_se:
9532 case Intrinsic::riscv_sf_vc_fv_se:
9534 case Intrinsic::riscv_sf_vc_xvv_se:
9536 case Intrinsic::riscv_sf_vc_ivv_se:
9538 case Intrinsic::riscv_sf_vc_vvv_se:
9540 case Intrinsic::riscv_sf_vc_fvv_se:
9542 case Intrinsic::riscv_sf_vc_xvw_se:
9544 case Intrinsic::riscv_sf_vc_ivw_se:
9546 case Intrinsic::riscv_sf_vc_vvw_se:
9548 case Intrinsic::riscv_sf_vc_fvw_se:
9556 switch (ISDOpcode) {
9559 case ISD::VP_REDUCE_ADD:
9562 case ISD::VP_REDUCE_UMAX:
9565 case ISD::VP_REDUCE_SMAX:
9568 case ISD::VP_REDUCE_UMIN:
9571 case ISD::VP_REDUCE_SMIN:
9574 case ISD::VP_REDUCE_AND:
9577 case ISD::VP_REDUCE_OR:
9580 case ISD::VP_REDUCE_XOR:
9583 case ISD::VP_REDUCE_FADD:
9585 case ISD::VP_REDUCE_SEQ_FADD:
9587 case ISD::VP_REDUCE_FMAX:
9588 case ISD::VP_REDUCE_FMAXIMUM:
9590 case ISD::VP_REDUCE_FMIN:
9591 case ISD::VP_REDUCE_FMINIMUM:
9601 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9606 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9607 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9608 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9609 "Unexpected reduction lowering");
9613 MVT ContainerVT = VecVT;
9622 VL =
Op.getOperand(3);
9624 std::tie(Mask, VL) =
9632 switch (
Op.getOpcode()) {
9636 case ISD::VP_REDUCE_AND: {
9646 case ISD::VP_REDUCE_OR:
9653 case ISD::VP_REDUCE_XOR: {
9676 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9680 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9681 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9682 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9683 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9699 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9703 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9706 if (M1VT != InnerVT)
9712 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9731 VecEVT =
Lo.getValueType();
9744 MVT ContainerVT = VecVT;
9764 Mask, VL,
DL, DAG, Subtarget);
9770static std::tuple<unsigned, SDValue, SDValue>
9774 auto Flags =
Op->getFlags();
9775 unsigned Opcode =
Op.getOpcode();
9799 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9807 MVT VecEltVT =
Op.getSimpleValueType();
9811 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9815 MVT ContainerVT = VecVT;
9821 MVT ResVT =
Op.getSimpleValueType();
9824 VL,
DL, DAG, Subtarget);
9829 if (
Op->getFlags().hasNoNaNs())
9835 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9836 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9842 DL, ResVT, NoNaNs, Res,
9850 unsigned Opc =
Op.getOpcode();
9873 Vec, Mask, VL,
DL, DAG, Subtarget);
9874 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
9875 Op->getFlags().hasNoNaNs())
9892 DL, ResVT, NoNaNs, Res,
9906 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9915 (OrigIdx != 0 || !Vec.
isUndef())) {
9918 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9921 "Unexpected mask vector lowering");
9955 MVT ContainerVT = VecVT;
9991 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9992 SlideupAmt, Mask, VL, Policy);
10000 MVT ContainerVecVT = VecVT;
10006 MVT ContainerSubVecVT = SubVecVT;
10012 unsigned SubRegIdx;
10022 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10023 SubRegIdx = Decompose.first;
10025 (OrigIdx % Vscale));
10029 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
10030 SubRegIdx = Decompose.first;
10037 bool ExactlyVecRegSized =
10039 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
10054 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10058 if (SubRegIdx == RISCV::NoSubRegister) {
10077 MVT InterSubVT = ContainerVecVT;
10078 SDValue AlignedExtract = Vec;
10116 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10117 SlideupAmt, Mask, VL, Policy);
10122 if (ContainerVecVT.
bitsGT(InterSubVT))
10131 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10137 MVT SubVecVT =
Op.getSimpleValueType();
10142 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10153 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10156 "Unexpected mask vector lowering");
10195 MVT ContainerVT = VecVT;
10203 if (
auto ShrunkVT =
10205 ContainerVT = *ShrunkVT;
10219 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10231 MVT ContainerSubVecVT = SubVecVT;
10235 unsigned SubRegIdx;
10245 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10246 SubRegIdx = Decompose.first;
10248 (OrigIdx % Vscale));
10252 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10253 SubRegIdx = Decompose.first;
10276 MVT InterSubVT = VecVT;
10280 assert(SubRegIdx != RISCV::NoSubRegister);
10294 Vec, SlidedownAmt, Mask, VL);
10303 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10310 MVT VT =
N.getSimpleValueType();
10314 assert(
Op.getSimpleValueType() == VT &&
10315 "Operands and result must be same type");
10319 unsigned NumVals =
N->getNumValues();
10322 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10325 for (
unsigned I = 0;
I < NumVals;
I++) {
10331 if (TruncVals.
size() > 1)
10333 return TruncVals.
front();
10339 MVT VecVT =
Op.getSimpleValueType();
10342 "vector_interleave on non-scalable vector!");
10353 EVT SplitVT = Op0Lo.getValueType();
10356 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10358 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10372 Op.getOperand(0),
Op.getOperand(1));
10399 Concat, EvenIdx, Passthru, Mask, VL);
10401 Concat, OddIdx, Passthru, Mask, VL);
10415 MVT VecVT =
Op.getSimpleValueType();
10418 "vector_interleave on non-scalable vector!");
10431 EVT SplitVT = Op0Lo.getValueType();
10434 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10436 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10458 Op.getOperand(0),
Op.getOperand(1));
10506 MVT VT =
Op.getSimpleValueType();
10511 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10512 if (StepValImm != 1) {
10521 VL, VT,
DL, DAG, Subtarget);
10536 MVT VecVT =
Op.getSimpleValueType();
10546 unsigned MaxVLMAX =
10556 if (MaxVLMAX > 256 && EltSize == 8) {
10603 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10613 MVT VecVT =
Op.getSimpleValueType();
10617 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10618 SDValue DownOffset, UpOffset;
10619 if (ImmValue >= 0) {
10635 DownOffset, TrueMask, UpOffset);
10636 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10642RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10645 auto *
Load = cast<LoadSDNode>(
Op);
10648 Load->getMemoryVT(),
10649 *
Load->getMemOperand()) &&
10650 "Expecting a correctly-aligned load");
10652 MVT VT =
Op.getSimpleValueType();
10658 const auto [MinVLMAX, MaxVLMAX] =
10661 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10675 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10684 Load->getMemoryVT(),
Load->getMemOperand());
10691RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10694 auto *
Store = cast<StoreSDNode>(
Op);
10697 Store->getMemoryVT(),
10698 *
Store->getMemOperand()) &&
10699 "Expecting a correctly-aligned store");
10721 const auto [MinVLMAX, MaxVLMAX] =
10724 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10736 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10739 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10740 Store->getMemoryVT(),
Store->getMemOperand());
10746 MVT VT =
Op.getSimpleValueType();
10748 const auto *MemSD = cast<MemSDNode>(
Op);
10749 EVT MemVT = MemSD->getMemoryVT();
10751 SDValue Chain = MemSD->getChain();
10755 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10756 Mask = VPLoad->getMask();
10758 VL = VPLoad->getVectorLength();
10760 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10761 Mask = MLoad->getMask();
10762 PassThru = MLoad->getPassThru();
10769 MVT ContainerVT = VT;
10783 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10800 Chain =
Result.getValue(1);
10812 const auto *MemSD = cast<MemSDNode>(
Op);
10813 EVT MemVT = MemSD->getMemoryVT();
10815 SDValue Chain = MemSD->getChain();
10819 bool IsCompressingStore =
false;
10820 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10821 Val = VPStore->getValue();
10822 Mask = VPStore->getMask();
10823 VL = VPStore->getVectorLength();
10825 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10826 Val = MStore->getValue();
10827 Mask = MStore->getMask();
10828 IsCompressingStore = MStore->isCompressingStore();
10837 MVT ContainerVT = VT;
10842 if (!IsUnmasked || IsCompressingStore) {
10851 if (IsCompressingStore) {
10854 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10861 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10870 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10874RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10876 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10879 MVT VT =
Op.getSimpleValueType();
10893 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10900 unsigned Opc =
Op.getOpcode();
10907 MVT VT =
Op.getSimpleValueType();
10940 MVT ContainerInVT = InVT;
10959 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10963 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10971 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10976 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10989 MVT VT =
Op.getSimpleValueType();
10993 "Unexpected type for ISD::ABS");
10995 MVT ContainerVT = VT;
11002 if (
Op->getOpcode() == ISD::VP_ABS) {
11003 Mask =
Op->getOperand(1);
11007 VL =
Op->getOperand(2);
11015 DAG.
getUNDEF(ContainerVT), Mask, VL);
11017 DAG.
getUNDEF(ContainerVT), Mask, VL);
11024SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
11027 MVT VT =
Op.getSimpleValueType();
11031 "Can only handle COPYSIGN with matching types.");
11040 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
11045SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
11047 MVT VT =
Op.getSimpleValueType();
11050 MVT I1ContainerVT =
11064 Op2, DAG.
getUNDEF(ContainerVT), VL);
11075 MVT VT =
Op.getSimpleValueType();
11080 for (
const SDValue &V :
Op->op_values()) {
11081 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11084 if (!
V.getValueType().isVector()) {
11090 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
11091 "Only fixed length vectors are supported!");
11105 if (
Op->isStrictFPOpcode()) {
11114 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11128 MVT VT =
Op.getSimpleValueType();
11131 MVT ContainerVT = VT;
11137 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11143 if (*MaskIdx == OpIdx.index())
11147 if (
Op.getOpcode() == ISD::VP_MERGE) {
11151 assert(
Op.getOpcode() == ISD::VP_SELECT);
11158 if (!
V.getValueType().isFixedLengthVector()) {
11163 MVT OpVT =
V.getSimpleValueType();
11165 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11166 "Only fixed length vectors are supported!");
11171 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11181 MVT VT =
Op.getSimpleValueType();
11187 MVT ContainerVT = VT;
11197 DAG.
getUNDEF(ContainerVT), Zero, VL);
11200 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11202 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11205 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11214 MVT VT =
Op.getSimpleValueType();
11218 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11222 MVT ContainerVT = VT;
11232 switch (Condition) {
11300 MVT DstVT =
Op.getSimpleValueType();
11301 MVT SrcVT = Src.getSimpleValueType();
11314 if (DstEltSize >= SrcEltSize) {
11323 if (SrcEltSize == 1) {
11334 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11335 }
else if (DstEltSize > (2 * SrcEltSize)) {
11339 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11345 "Wrong input/output vector types");
11348 if (DstEltSize > (2 * SrcEltSize)) {
11364 MVT InterimFVT = DstVT;
11365 if (SrcEltSize > (2 * DstEltSize)) {
11366 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11373 if (InterimFVT != DstVT) {
11379 "Wrong input/output vector types");
11383 if (DstEltSize == 1) {
11386 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11396 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11406 while (InterimIVT != DstVT) {
11418 MVT VT =
Op.getSimpleValueType();
11425RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11437 MVT VT =
Op.getSimpleValueType();
11438 MVT ContainerVT = VT;
11451 if (IsMaskVector) {
11462 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11471 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11474 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11475 SDValue DownOffset, UpOffset;
11476 if (ImmValue >= 0) {
11490 Op1, DownOffset, Mask, UpOffset);
11494 if (IsMaskVector) {
11498 {Result, DAG.getConstant(0, DL, ContainerVT),
11499 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11509RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11512 MVT VT =
Op.getSimpleValueType();
11519 MVT ContainerVT = VT;
11527 MVT GatherVT = ContainerVT;
11531 if (IsMaskVector) {
11542 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11548 unsigned MaxVLMAX =
11557 if (MaxVLMAX > 256 && EltSize == 8) {
11585 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11587 if (IsMaskVector) {
11610 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11612 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11614 DAG.
getUNDEF(GatherVT), Mask, EVL);
11616 if (IsMaskVector) {
11631 MVT VT =
Op.getSimpleValueType();
11633 return lowerVPOp(
Op, DAG);
11640 MVT ContainerVT = VT;
11659 MVT VT =
Op.getSimpleValueType();
11660 MVT ContainerVT = VT;
11666 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11672 : Intrinsic::riscv_vlse_mask,
11675 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11676 VPNode->getStride()};
11684 Ops.
push_back(VPNode->getVectorLength());
11692 VPNode->getMemoryVT(), VPNode->getMemOperand());
11706 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11707 SDValue StoreVal = VPNode->getValue();
11709 MVT ContainerVT = VT;
11720 : Intrinsic::riscv_vsse_mask,
11723 VPNode->getBasePtr(), VPNode->getStride()};
11731 Ops.
push_back(VPNode->getVectorLength());
11734 Ops, VPNode->getMemoryVT(),
11735 VPNode->getMemOperand());
11747 MVT VT =
Op.getSimpleValueType();
11749 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11750 EVT MemVT = MemSD->getMemoryVT();
11752 SDValue Chain = MemSD->getChain();
11758 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11759 Index = VPGN->getIndex();
11760 Mask = VPGN->getMask();
11762 VL = VPGN->getVectorLength();
11767 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11768 Index = MGN->getIndex();
11769 Mask = MGN->getMask();
11770 PassThru = MGN->getPassThru();
11774 MVT IndexVT =
Index.getSimpleValueType();
11778 "Unexpected VTs!");
11779 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11782 "Unexpected extending MGATHER/VP_GATHER");
11788 MVT ContainerVT = VT;
11812 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11829 Chain =
Result.getValue(1);
11846 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11847 EVT MemVT = MemSD->getMemoryVT();
11849 SDValue Chain = MemSD->getChain();
11852 [[maybe_unused]]
bool IsTruncatingStore =
false;
11855 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11856 Index = VPSN->getIndex();
11857 Mask = VPSN->getMask();
11858 Val = VPSN->getValue();
11859 VL = VPSN->getVectorLength();
11861 IsTruncatingStore =
false;
11864 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11865 Index = MSN->getIndex();
11866 Mask = MSN->getMask();
11867 Val = MSN->getValue();
11868 IsTruncatingStore = MSN->isTruncatingStore();
11872 MVT IndexVT =
Index.getSimpleValueType();
11876 "Unexpected VTs!");
11877 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11880 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11886 MVT ContainerVT = VT;
11910 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11920 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11929 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11937 static const int Table =
11961 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11967 static const unsigned Table =
11990 bool isRISCV64 = Subtarget.
is64Bit();
12054 switch (
N->getOpcode()) {
12056 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12062 "Unexpected custom legalisation");
12063 bool IsStrict =
N->isStrictFPOpcode();
12066 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12084 Opc,
DL, VTs, Chain, Op0,
12118 std::tie(Result, Chain) =
12119 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12147 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12158 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12159 "has custom type legalization on riscv32");
12161 SDValue LoCounter, HiCounter;
12165 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12167 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12170 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12172 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12176 N->getOperand(0), LoCounter, HiCounter);
12200 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12201 unsigned XLen = Subtarget.
getXLen();
12204 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12212 if (LHSIsU == RHSIsU)
12229 if (RHSIsU && LHSIsS && !RHSIsS)
12231 else if (LHSIsU && RHSIsS && !LHSIsS)
12241 "Unexpected custom legalisation");
12248 "Unexpected custom legalisation");
12251 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12277 "Unexpected custom legalisation");
12278 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12279 Subtarget.hasVendorXTHeadBb()) &&
12280 "Unexpected custom legalization");
12281 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12282 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12291 "Unexpected custom legalisation");
12305 MVT VT =
N->getSimpleValueType(0);
12306 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12307 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12308 "Unexpected custom legalisation");
12320 if (VT != MVT::i32)
12329 "Unexpected custom legalisation");
12333 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12350 EVT OType =
N->getValueType(1);
12363 "Unexpected custom legalisation");
12380 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12384 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12402 "Unexpected custom legalisation");
12403 if (Subtarget.hasStdExtZbb()) {
12424 "Unexpected custom legalisation");
12430 "Unexpected custom legalisation");
12432 if (Subtarget.hasStdExtZbb()) {
12466 EVT VT =
N->getValueType(0);
12471 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12475 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12476 Subtarget.hasStdExtZfbfmin()) {
12479 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12484 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12486 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12507 MVT VT =
N->getSimpleValueType(0);
12509 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12510 "Unexpected custom legalisation");
12513 "Unexpected extension");
12539 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12541 "Unexpected EXTRACT_VECTOR_ELT legalization");
12544 MVT ContainerVT = VecVT;
12572 DAG.
getUNDEF(ContainerVT), Mask, VL);
12580 unsigned IntNo =
N->getConstantOperandVal(0);
12584 "Don't know how to custom type legalize this intrinsic!");
12585 case Intrinsic::experimental_get_vector_length: {
12590 case Intrinsic::experimental_cttz_elts: {
12596 case Intrinsic::riscv_orc_b:
12597 case Intrinsic::riscv_brev8:
12598 case Intrinsic::riscv_sha256sig0:
12599 case Intrinsic::riscv_sha256sig1:
12600 case Intrinsic::riscv_sha256sum0:
12601 case Intrinsic::riscv_sha256sum1:
12602 case Intrinsic::riscv_sm3p0:
12603 case Intrinsic::riscv_sm3p1: {
12604 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12624 case Intrinsic::riscv_sm4ks:
12625 case Intrinsic::riscv_sm4ed: {
12633 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12637 case Intrinsic::riscv_mopr: {
12638 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12648 case Intrinsic::riscv_moprr: {
12649 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12661 case Intrinsic::riscv_clmul: {
12662 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12673 case Intrinsic::riscv_clmulh:
12674 case Intrinsic::riscv_clmulr: {
12675 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12703 case Intrinsic::riscv_vmv_x_s: {
12704 EVT VT =
N->getValueType(0);
12706 if (VT.
bitsLT(XLenVT)) {
12715 "Unexpected custom legalization");
12753 case ISD::VP_REDUCE_ADD:
12754 case ISD::VP_REDUCE_AND:
12755 case ISD::VP_REDUCE_OR:
12756 case ISD::VP_REDUCE_XOR:
12757 case ISD::VP_REDUCE_SMAX:
12758 case ISD::VP_REDUCE_UMAX:
12759 case ISD::VP_REDUCE_SMIN:
12760 case ISD::VP_REDUCE_UMIN:
12824 const EVT VT =
N->getValueType(0);
12825 const unsigned Opc =
N->getOpcode();
12832 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12837 "Inconsistent mappings");
12848 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12851 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12866 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12868 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12869 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12873 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12880 if (
LHS.getOpcode() != ReduceOpc)
12895 auto Flags = ReduceVec->
getFlags();
12896 Flags.intersectWith(
N->getFlags());
12897 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12907 auto BinOpToRVVReduce = [](
unsigned Opc) {
12936 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12939 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12942 unsigned Opc =
N->getOpcode();
12943 unsigned ReduceIdx;
12944 if (IsReduction(
N->getOperand(0), Opc))
12946 else if (IsReduction(
N->getOperand(1), Opc))
12952 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12955 SDValue Extract =
N->getOperand(ReduceIdx);
12987 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
13014 if (!Subtarget.hasStdExtZba())
13018 EVT VT =
N->getValueType(0);
13030 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13031 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
13034 int64_t C0 = N0C->getSExtValue();
13035 int64_t C1 = N1C->getSExtValue();
13036 if (C0 <= 0 || C1 <= 0)
13040 int64_t Bits = std::min(C0, C1);
13041 int64_t Diff = std::abs(C0 - C1);
13042 if (Diff != 1 && Diff != 2 && Diff != 3)
13069 EVT VT =
N->getValueType(0);
13077 if ((!Subtarget.hasStdExtZicond() &&
13078 !Subtarget.hasVendorXVentanaCondOps()) ||
13100 bool SwapSelectOps;
13106 SwapSelectOps =
false;
13107 NonConstantVal = FalseVal;
13109 SwapSelectOps =
true;
13110 NonConstantVal = TrueVal;
13116 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13164 EVT VT =
N->getValueType(0);
13172 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13173 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13179 if (!N0C->hasOneUse())
13181 int64_t C0 = N0C->getSExtValue();
13182 int64_t C1 = N1C->getSExtValue();
13184 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13187 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
13188 !isInt<12>(C0 * (C1 / C0))) {
13191 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
13192 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13195 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13196 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13223 EVT VT =
N->getValueType(0);
13254 unsigned OuterExtend =
13258 OuterExtend,
SDLoc(
N), VT,
13266 EVT VT =
N->getValueType(0);
13314 EVT VT =
N->getValueType(0);
13318 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13324 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13334 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13355 if (!Subtarget.hasStdExtZbb())
13358 EVT VT =
N->getValueType(0);
13360 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
13369 auto *ShAmtC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
13370 if (!ShAmtC || ShAmtC->getZExtValue() != 8)
13385 EVT VT =
N->getValueType(0);
13416 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13440 EVT VT =
N->getValueType(0);
13464 EVT VT =
N->getValueType(0);
13491 if (CondLHS != True)
13498 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
13510 if (!FalseRHSC || !FalseRHSC->
isZero())
13530 EVT VT =
N->getValueType(0);
13616 EVT VT =
N->getValueType(0);
13687 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13692 const APInt &Imm = ConstN00->getAPIntValue();
13693 if ((Imm + 1).isSignedIntN(12))
13710 LHS.getValueType());
13731 EVT VT =
N->getValueType(0);
13743 const bool HasShlAdd =
13744 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
13758 for (
uint64_t Divisor : {3, 5, 9}) {
13759 if (MulAmt % Divisor != 0)
13761 uint64_t MulAmt2 = MulAmt / Divisor;
13768 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
13769 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
13786 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13802 if (ScaleShift >= 1 && ScaleShift < 4) {
13803 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13817 for (
uint64_t Divisor : {3, 5, 9}) {
13822 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13833 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13835 if (ScaleShift >= 1 && ScaleShift < 4) {
13836 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13862 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
13864 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
13881 EVT VT =
N->getValueType(0);
13888 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
13889 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
13902 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
13903 V3 != (HalfSize - 1))
13919 EVT VT =
N->getValueType(0);
13927 unsigned AddSubOpc;
13933 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13934 AddSubOpc = V->getOpcode();
13936 SDValue Opnd = V->getOperand(1);
13937 MulOper = V->getOperand(0);
13946 if (IsAddSubWith1(N0)) {
13948 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13951 if (IsAddSubWith1(N1)) {
13953 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13968 if (isIndexTypeSigned(IndexType))
13971 if (!
N->hasOneUse())
13974 EVT VT =
N.getValueType();
14013 EVT SrcVT = Src.getValueType();
14017 NewElen = std::max(NewElen, 8U);
14040 EVT VT =
N->getValueType(0);
14043 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
14047 auto *N1C = dyn_cast<ConstantSDNode>(N1);
14059 if (!isIntEqualitySetCC(
Cond))
14068 const APInt &C1 = N1C->getAPIntValue();
14086 EVT VT =
N->getValueType(0);
14090 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
14092 Src.getOperand(0));
14100struct CombineResult;
14102enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
14129struct NodeExtensionHelper {
14138 bool SupportsFPExt;
14141 bool EnforceOneUse;
14156 return OrigOperand;
14167 unsigned getExtOpc(ExtKind SupportsExt)
const {
14168 switch (SupportsExt) {
14169 case ExtKind::SExt:
14171 case ExtKind::ZExt:
14173 case ExtKind::FPExt:
14184 std::optional<ExtKind> SupportsExt)
const {
14185 if (!SupportsExt.has_value())
14186 return OrigOperand;
14188 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
14192 if (
Source.getValueType() == NarrowVT)
14195 unsigned ExtOpc = getExtOpc(*SupportsExt);
14199 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
14206 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
14224 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
14230 MVT EltVT = SupportsExt == ExtKind::FPExt
14232 :
MVT::getIntegerVT(NarrowSize);
14234 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
14235 "Trying to extend something we can't represent");
14242 static unsigned getSExtOpcode(
unsigned Opcode) {
14265 static unsigned getZExtOpcode(
unsigned Opcode) {
14291 static unsigned getFPExtOpcode(
unsigned Opcode) {
14308 static unsigned getSUOpcode(
unsigned Opcode) {
14310 "SU is only supported for MUL");
14316 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14336 using CombineToTry = std::function<std::optional<CombineResult>(
14337 SDNode * ,
const NodeExtensionHelper & ,
14342 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14346 unsigned Opc = OrigOperand.
getOpcode();
14350 "Unexpected Opcode");
14363 unsigned ScalarBits =
Op.getValueSizeInBits();
14366 if (ScalarBits < EltBits)
14372 if (NarrowSize < 8)
14376 SupportsSExt =
true;
14380 SupportsZExt =
true;
14382 EnforceOneUse =
false;
14389 SupportsZExt =
false;
14390 SupportsSExt =
false;
14391 SupportsFPExt =
false;
14392 EnforceOneUse =
true;
14393 unsigned Opc = OrigOperand.
getOpcode();
14415 SupportsZExt =
true;
14418 SupportsSExt =
true;
14421 SupportsFPExt =
true;
14425 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14433 static bool isSupportedRoot(
const SDNode *Root,
14462 Subtarget.hasStdExtZvbb();
14464 return Subtarget.hasStdExtZvbb();
14473 assert(isSupportedRoot(Root, Subtarget) &&
14474 "Trying to build an helper with an "
14475 "unsupported root");
14476 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14492 if (OperandIdx == 1) {
14501 EnforceOneUse =
false;
14506 fillUpExtensionSupport(Root, DAG, Subtarget);
14512 static std::pair<SDValue, SDValue>
14515 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14534 switch (
N->getOpcode()) {
14572struct CombineResult {
14574 unsigned TargetOpcode;
14576 std::optional<ExtKind> LHSExt;
14577 std::optional<ExtKind> RHSExt;
14581 NodeExtensionHelper
LHS;
14583 NodeExtensionHelper
RHS;
14585 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14586 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14587 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14588 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14597 std::tie(Mask, VL) =
14598 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14612 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14613 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14628static std::optional<CombineResult>
14629canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14630 const NodeExtensionHelper &RHS,
14633 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14634 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14635 Root, LHS, {ExtKind::ZExt}, RHS,
14637 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14638 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14639 Root, LHS, {ExtKind::SExt}, RHS,
14641 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14642 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14643 Root, LHS, {ExtKind::FPExt}, RHS,
14645 return std::nullopt;
14654static std::optional<CombineResult>
14655canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14658 return canFoldToVWWithSameExtensionImpl(
14659 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14667static std::optional<CombineResult>
14668canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14671 if (
RHS.SupportsFPExt)
14672 return CombineResult(
14673 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14674 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14681 return CombineResult(
14682 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14683 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14685 return CombineResult(
14686 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14687 LHS, std::nullopt, RHS, {ExtKind::SExt});
14688 return std::nullopt;
14695static std::optional<CombineResult>
14696canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14699 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14707static std::optional<CombineResult>
14708canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14711 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14719static std::optional<CombineResult>
14720canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14723 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14731static std::optional<CombineResult>
14732canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14736 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14737 return std::nullopt;
14738 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14739 Root, LHS, {ExtKind::SExt}, RHS,
14744NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14755 Strategies.
push_back(canFoldToVWWithSameExtension);
14760 Strategies.
push_back(canFoldToVWWithSameExtension);
14765 Strategies.
push_back(canFoldToVWWithSameExtension);
14772 Strategies.
push_back(canFoldToVWWithZEXT);
14777 Strategies.
push_back(canFoldToVWWithSEXT);
14782 Strategies.
push_back(canFoldToVWWithZEXT);
14787 Strategies.
push_back(canFoldToVWWithFPEXT);
14816 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14822 Inserted.insert(
N);
14825 while (!Worklist.
empty()) {
14827 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14830 NodeExtensionHelper
LHS(
N, 0, DAG, Subtarget);
14831 NodeExtensionHelper
RHS(
N, 1, DAG, Subtarget);
14832 auto AppendUsersIfNeeded = [&Worklist,
14833 &Inserted](
const NodeExtensionHelper &
Op) {
14834 if (
Op.needToPromoteOtherUsers()) {
14835 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14836 if (Inserted.insert(TheUse).second)
14848 NodeExtensionHelper::getSupportedFoldings(
N);
14850 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14851 bool Matched =
false;
14852 for (
int Attempt = 0;
14853 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
14856 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14857 FoldingStrategies) {
14858 std::optional<CombineResult> Res =
14859 FoldingStrategy(
N,
LHS,
RHS, DAG, Subtarget);
14866 if (Res->LHSExt.has_value())
14867 AppendUsersIfNeeded(
LHS);
14868 if (Res->RHSExt.has_value())
14869 AppendUsersIfNeeded(
RHS);
14880 SDValue InputRootReplacement;
14887 for (CombineResult Res : CombinesToApply) {
14888 SDValue NewValue = Res.materialize(DAG, Subtarget);
14889 if (!InputRootReplacement) {
14891 "First element is expected to be the current node");
14892 InputRootReplacement = NewValue;
14897 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14901 return InputRootReplacement;
14908 unsigned Opc =
N->getOpcode();
14913 SDValue MergeOp =
N->getOperand(1);
14914 unsigned MergeOpc = MergeOp.
getOpcode();
14925 SDValue Passthru =
N->getOperand(2);
14939 Z = Z.getOperand(1);
14945 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14952 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14981 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14987 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14989 if (MemVT == MVT::i32)
14995 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
15030 if (!Subtarget.hasVendorXTHeadMemPair())
15042 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
15044 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
15045 return {
Ptr->getOperand(0), C1->getZExtValue()};
15049 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
15072 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
15075 if (Base1 != Base2)
15079 bool Valid =
false;
15080 if (MemVT == MVT::i32) {
15082 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
15084 }
else if (MemVT == MVT::i64) {
15086 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
15120 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15128 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15138 EVT VT =
N->getValueType(0);
15141 MVT SrcVT = Src.getSimpleValueType();
15142 MVT SrcContainerVT = SrcVT;
15144 SDValue XVal = Src.getOperand(0);
15171 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15175 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15179 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
15192 if (VT != MVT::i32 && VT != XLenVT)
15222 EVT DstVT =
N->getValueType(0);
15223 if (DstVT != XLenVT)
15229 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15237 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15240 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
15249 if (SatVT == DstVT)
15251 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
15257 Src = Src.getOperand(0);
15278 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
15284 EVT VT =
N->getValueType(0);
15337 unsigned Offset =
N->isTargetStrictFPOpcode();
15344 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15346 V.getOperand(2) == VL) {
15348 V = V.getOperand(0);
15355 bool NegA = invertIfNegative(
A);
15356 bool NegB = invertIfNegative(
B);
15357 bool NegC = invertIfNegative(
C);
15360 if (!NegA && !NegB && !NegC)
15364 if (
N->isTargetStrictFPOpcode())
15366 {N->getOperand(0), A, B, C, Mask, VL});
15376 if (
N->getValueType(0).isScalableVector() &&
15377 N->getValueType(0).getVectorElementType() == MVT::f32 &&
15384 if (
N->isTargetStrictFPOpcode())
15409 switch (
N->getOpcode()) {
15429 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15430 N->getOperand(2), Mask, VL);
15437 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15440 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15442 uint64_t ShAmt =
N->getConstantOperandVal(1);
15453 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15481 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15494 !isa<ConstantSDNode>(U->getOperand(1)) ||
15495 U->getConstantOperandVal(1) > 32)
15550 if (!
Cond.hasOneUse())
15569 EVT VT =
Cond.getValueType();
15614 LHS =
LHS.getOperand(0);
15624 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15628 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15632 RHS =
LHS.getOperand(1);
15633 LHS =
LHS.getOperand(0);
15642 RHS =
LHS.getOperand(1);
15643 LHS =
LHS.getOperand(0);
15659 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15700 bool Commutative =
true;
15701 unsigned Opc = TrueVal.getOpcode();
15709 Commutative =
false;
15717 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15721 if (FalseVal == TrueVal.getOperand(0))
15723 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15728 EVT VT =
N->getValueType(0);
15730 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15736 assert(IdentityOperand &&
"No identity operand!");
15741 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15742 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15763 CountZeroes =
N->getOperand(2);
15764 ValOnZero =
N->getOperand(1);
15766 CountZeroes =
N->getOperand(1);
15767 ValOnZero =
N->getOperand(2);
15786 if (
Cond->getOperand(0) != CountZeroesArgument)
15802 CountZeroes, BitWidthMinusOne);
15812 EVT VT =
N->getValueType(0);
15813 EVT CondVT =
Cond.getValueType();
15821 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15827 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15848 SDValue TrueVal =
N->getOperand(1);
15849 SDValue FalseVal =
N->getOperand(2);
15864 EVT VT =
N->getValueType(0);
15871 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15886 if (
Op.isUndef()) {
15899 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15903 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15904 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15908 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15936 const unsigned InVecOpcode = InVec->
getOpcode();
15946 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15953 InVecLHS, InValLHS, EltNo);
15955 InVecRHS, InValRHS, EltNo);
15964 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15967 unsigned Elt = IndexC->getZExtValue();
15975 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15978 ConcatOp, InVal, NewIdx);
15982 ConcatOps[ConcatOpIdx] = ConcatOp;
15994 EVT VT =
N->getValueType(0);
16004 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
16006 !
SDValue(BaseLd, 0).hasOneUse())
16009 EVT BaseLdVT = BaseLd->getValueType(0);
16016 auto *Ld = dyn_cast<LoadSDNode>(
Op);
16017 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
16019 Ld->getValueType(0) != BaseLdVT)
16028 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
16030 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
16035 if (BIO1.equalBaseIndex(BIO2, DAG))
16040 SDValue P2 = Ld2->getBasePtr();
16046 return std::nullopt;
16050 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
16055 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
16056 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
16064 unsigned WideScalarBitWidth =
16077 auto [StrideVariant, MustNegateStride] = *BaseDiff;
16078 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
16079 ? std::get<SDValue>(StrideVariant)
16082 if (MustNegateStride)
16095 BaseLd->getBasePtr(), Stride, AllOneMask};
16098 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
16099 ConstStride && ConstStride->getSExtValue() >= 0)
16103 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
16109 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
16113 Ops, WideVecVT, MMO);
16125 if (
N->getValueType(0).isFixedLengthVector())
16128 SDValue Addend =
N->getOperand(0);
16132 SDValue AddMergeOp =
N->getOperand(2);
16137 auto IsVWMulOpc = [](
unsigned Opc) {
16166 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
16167 }(
N, DAG, Subtarget);
16172 if (AddMask != MulMask || AddVL != MulVL)
16177 "Unexpected opcode after VWMACC_VL");
16179 "Unexpected opcode after VWMACC_VL!");
16181 "Unexpected opcode after VWMUL_VL!");
16183 "Unexpected opcode after VWMUL_VL!");
16186 EVT VT =
N->getValueType(0);
16202 const EVT IndexVT =
Index.getValueType();
16206 if (!isIndexTypeSigned(IndexType))
16238 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16241 if (
Index->getOperand(i)->isUndef())
16244 if (
C % ElementSize != 0)
16246 C =
C / ElementSize;
16250 ActiveLanes.
set(
C);
16252 return ActiveLanes.
all();
16270 if (NumElems % 2 != 0)
16274 const unsigned WiderElementSize = ElementSize * 2;
16275 if (WiderElementSize > ST.getELen()/8)
16278 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
16281 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16284 if (
Index->getOperand(i)->isUndef())
16290 if (
C % WiderElementSize != 0)
16295 if (
C !=
Last + ElementSize)
16312 (isa<RegisterSDNode>(VL) &&
16313 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16315 Mask.getOperand(0) != VL)
16318 auto IsTruncNode = [&](
SDValue V) {
16320 V.getOperand(1) == Mask && V.getOperand(2) == VL;
16327 while (IsTruncNode(
Op)) {
16328 if (!
Op.hasOneUse())
16330 Op =
Op.getOperand(0);
16365 MVT VT =
N->getSimpleValueType(0);
16370 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
16372 if (V.getOpcode() != Opc &&
16373 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
16374 V.getOperand(3) == Mask && V.getOperand(4) == VL))
16382 Op.getOperand(1).getValueType().isFixedLengthVector() &&
16384 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
16386 Op =
Op.getOperand(1).getOperand(0);
16389 return V.getOperand(0);
16392 Op.getOperand(2) == VL) {
16393 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
16395 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
16396 return V.getOperand(0);
16405 auto DetectUSatPattern = [&](
SDValue V) {
16430 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
16436 auto DetectSSatPattern = [&](
SDValue V) {
16438 unsigned NumSrcBits = V.getScalarValueSizeInBits();
16446 if (HiC == SignedMax && LoC == SignedMin)
16452 if (HiC == SignedMax && LoC == SignedMin)
16462 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
16464 Src = Src.getOperand(0);
16468 if ((Val = DetectUSatPattern(Src)))
16470 else if ((Val = DetectSSatPattern(Src)))
16482 ClipOpc,
DL, ValVT,
16486 }
while (ValVT != VT);
16500 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16511 switch (
N->getOpcode()) {
16531 APInt V =
C->getValueAPF().bitcastToAPInt();
16566 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16567 SimplifyDemandedLowBitsHelper(1, 5))
16575 if (SimplifyDemandedLowBitsHelper(0, 32))
16592 MVT VT =
N->getSimpleValueType(0);
16601 "Unexpected value type!");
16624 EVT VT =
N->getValueType(0);
16689 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16694 Src.getOperand(0));
16699 Src.getOperand(0), Src.getOperand(1));
16720 unsigned Opc =
N->getOpcode();
16735 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16745 N->getValueType(0), Val,
Cond.getOperand(0));
16756 SDValue FalseV =
N->getOperand(4);
16758 EVT VT =
N->getValueType(0);
16761 if (TrueV == FalseV)
16766 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16772 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16773 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16776 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16777 isInt<12>(TrueSImm - FalseSImm)) {
16793 {LHS, RHS, CC, TrueV, FalseV});
16860 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16873 EVT VT =
N->getValueType(0);
16897 const auto *MGN = cast<MaskedGatherSDNode>(
N);
16898 const EVT VT =
N->getValueType(0);
16900 SDValue ScaleOp = MGN->getScale();
16902 assert(!MGN->isIndexScaled() &&
16903 "Scaled gather/scatter should not be formed");
16908 N->getVTList(), MGN->getMemoryVT(),
DL,
16909 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16910 MGN->getBasePtr(), Index, ScaleOp},
16911 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16915 N->getVTList(), MGN->getMemoryVT(),
DL,
16916 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16917 MGN->getBasePtr(), Index, ScaleOp},
16918 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16924 if (std::optional<VIDSequence> SimpleVID =
16926 SimpleVID && SimpleVID->StepDenominator == 1) {
16927 const int64_t StepNumerator = SimpleVID->StepNumerator;
16928 const int64_t Addend = SimpleVID->Addend;
16935 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16945 {MGN->getChain(), IntID, MGN->getPassThru(), BasePtr,
16946 DAG.
getConstant(StepNumerator,
DL, XLenVT), MGN->getMask()};
16948 Ops, VT, MGN->getMemOperand());
16956 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16958 MGN->getMemoryVT(), MGN->getMemOperand(),
16967 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16969 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16971 EVT IndexVT =
Index.getValueType()
16978 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16980 EltCnt.divideCoefficientBy(2));
16983 EltCnt.divideCoefficientBy(2));
16988 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16997 const auto *MSN = cast<MaskedScatterSDNode>(
N);
16999 SDValue ScaleOp = MSN->getScale();
17001 assert(!MSN->isIndexScaled() &&
17002 "Scaled gather/scatter should not be formed");
17007 N->getVTList(), MSN->getMemoryVT(),
DL,
17008 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17010 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17014 N->getVTList(), MSN->getMemoryVT(),
DL,
17015 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17017 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17019 EVT VT = MSN->getValue()->getValueType(0);
17021 if (!MSN->isTruncatingStore() &&
17025 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
17026 DAG.
getUNDEF(XLenVT), MSN->getMask(),
17027 MSN->getMemoryVT(), MSN->getMemOperand(),
17032 case ISD::VP_GATHER: {
17033 const auto *VPGN = cast<VPGatherSDNode>(
N);
17035 SDValue ScaleOp = VPGN->getScale();
17037 assert(!VPGN->isIndexScaled() &&
17038 "Scaled gather/scatter should not be formed");
17043 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17044 ScaleOp, VPGN->getMask(),
17045 VPGN->getVectorLength()},
17046 VPGN->getMemOperand(), IndexType);
17050 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17051 ScaleOp, VPGN->getMask(),
17052 VPGN->getVectorLength()},
17053 VPGN->getMemOperand(), IndexType);
17057 case ISD::VP_SCATTER: {
17058 const auto *VPSN = cast<VPScatterSDNode>(
N);
17060 SDValue ScaleOp = VPSN->getScale();
17062 assert(!VPSN->isIndexScaled() &&
17063 "Scaled gather/scatter should not be formed");
17068 {VPSN->getChain(), VPSN->getValue(),
17069 VPSN->getBasePtr(), Index, ScaleOp,
17070 VPSN->getMask(), VPSN->getVectorLength()},
17071 VPSN->getMemOperand(), IndexType);
17075 {VPSN->getChain(), VPSN->getValue(),
17076 VPSN->getBasePtr(), Index, ScaleOp,
17077 VPSN->getMask(), VPSN->getVectorLength()},
17078 VPSN->getMemOperand(), IndexType);
17092 EVT VT =
N->getValueType(0);
17095 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
17096 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
17114 EVT VT =
N->getValueType(0);
17118 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
17148 if (
N->getValueType(0).isScalableVector() &&
17149 N->getValueType(0).getVectorElementType() == MVT::f32 &&
17164 auto *Store = cast<StoreSDNode>(
N);
17165 SDValue Chain = Store->getChain();
17166 EVT MemVT = Store->getMemoryVT();
17167 SDValue Val = Store->getValue();
17170 bool IsScalarizable =
17172 Store->isSimple() &&
17202 NewVT, *Store->getMemOperand())) {
17204 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
17205 Store->getPointerInfo(), Store->getOriginalAlign(),
17206 Store->getMemOperand()->getFlags());
17214 if (
auto *L = dyn_cast<LoadSDNode>(Val);
17216 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
17218 L->getMemoryVT() == MemVT) {
17221 NewVT, *Store->getMemOperand()) &&
17223 NewVT, *L->getMemOperand())) {
17225 L->getPointerInfo(), L->getOriginalAlign(),
17226 L->getMemOperand()->getFlags());
17227 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
17228 Store->getPointerInfo(), Store->getOriginalAlign(),
17229 Store->getMemOperand()->getFlags());
17241 MVT VecVT = Src.getSimpleValueType();
17248 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
17251 Store->getMemOperand(), Store->getAddressingMode(),
17252 Store->isTruncatingStore(),
false);
17259 EVT VT =
N->getValueType(0);
17281 const MVT VT =
N->getSimpleValueType(0);
17282 SDValue Passthru =
N->getOperand(0);
17283 SDValue Scalar =
N->getOperand(1);
17292 const MVT VT =
N->getSimpleValueType(0);
17293 SDValue Passthru =
N->getOperand(0);
17294 SDValue Scalar =
N->getOperand(1);
17299 unsigned ScalarSize = Scalar.getValueSizeInBits();
17301 if (ScalarSize > EltWidth && Passthru.
isUndef())
17302 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
17309 (!Const || Const->isZero() ||
17310 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
17320 if (
N->getOperand(0).isUndef() &&
17323 Src.getOperand(0).getValueType().isScalableVector()) {
17324 EVT VT =
N->getValueType(0);
17325 EVT SrcVT = Src.getOperand(0).getValueType();
17329 return Src.getOperand(0);
17335 const MVT VT =
N->getSimpleValueType(0);
17336 SDValue Passthru =
N->getOperand(0);
17337 SDValue Scalar =
N->getOperand(1);
17341 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
17342 return Scalar.getOperand(0);
17351 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
17361 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
17369 MVT VecVT =
N->getOperand(0).getSimpleValueType();
17371 if (M1VT.
bitsLT(VecVT)) {
17382 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
17387 case Intrinsic::riscv_masked_strided_load: {
17388 MVT VT =
N->getSimpleValueType(0);
17389 auto *Load = cast<MemIntrinsicSDNode>(
N);
17390 SDValue PassThru =
N->getOperand(2);
17392 SDValue Stride =
N->getOperand(4);
17398 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
17399 StrideC && StrideC->getZExtValue() == ElementSize)
17401 DAG.
getUNDEF(XLenVT), Mask, PassThru,
17402 Load->getMemoryVT(), Load->getMemOperand(),
17406 case Intrinsic::riscv_masked_strided_store: {
17407 auto *Store = cast<MemIntrinsicSDNode>(
N);
17410 SDValue Stride =
N->getOperand(4);
17415 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
17416 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
17417 StrideC && StrideC->getZExtValue() == ElementSize)
17420 Value.getValueType(), Store->getMemOperand(),
17424 case Intrinsic::riscv_vcpop:
17425 case Intrinsic::riscv_vcpop_mask:
17426 case Intrinsic::riscv_vfirst:
17427 case Intrinsic::riscv_vfirst_mask: {
17429 if (IntNo == Intrinsic::riscv_vcpop_mask ||
17430 IntNo == Intrinsic::riscv_vfirst_mask)
17431 VL =
N->getOperand(3);
17436 EVT VT =
N->getValueType(0);
17437 if (IntNo == Intrinsic::riscv_vfirst ||
17438 IntNo == Intrinsic::riscv_vfirst_mask)
17447 EVT VT =
N->getValueType(0);
17451 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17470 EVT XVT,
unsigned KeptBits)
const {
17475 if (XVT != MVT::i32 && XVT != MVT::i64)
17479 if (KeptBits == 32 || KeptBits == 64)
17483 return Subtarget.hasStdExtZbb() &&
17484 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17492 "Expected shift op");
17503 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
17504 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17506 const APInt &C1Int = C1->getAPIntValue();
17507 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17533 if (C1Cost < ShiftedC1Cost)
17547 EVT VT =
Op.getValueType();
17551 unsigned Opcode =
Op.getOpcode();
17559 const APInt &Mask =
C->getAPIntValue();
17568 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17569 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17571 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17572 if (NewMask == Mask)
17577 Op.getOperand(0), NewC);
17590 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17591 if (IsLegalMask(NewMask))
17592 return UseMask(NewMask);
17595 if (VT == MVT::i64) {
17597 if (IsLegalMask(NewMask))
17598 return UseMask(NewMask);
17613 APInt NewMask = ShrunkMask;
17614 if (MinSignedBits <= 12)
17616 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17622 assert(IsLegalMask(NewMask));
17623 return UseMask(NewMask);
17627 static const uint64_t GREVMasks[] = {
17628 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17629 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17631 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17632 unsigned Shift = 1 << Stage;
17633 if (ShAmt & Shift) {
17635 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17647 const APInt &DemandedElts,
17649 unsigned Depth)
const {
17651 unsigned Opc =
Op.getOpcode();
17656 "Should use MaskedValueIsZero if you don't know whether Op"
17657 " is a target node!");
17740 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17743 if (MaxVLenB == MinVLenB)
17760 case Intrinsic::riscv_vsetvli:
17761 case Intrinsic::riscv_vsetvlimax: {
17762 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17763 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17769 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17772 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17773 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17775 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17788 unsigned Depth)
const {
17789 switch (
Op.getOpcode()) {
17795 if (Tmp == 1)
return 1;
17798 return std::min(Tmp, Tmp2);
17810 if (Tmp < 33)
return 1;
17835 unsigned XLen = Subtarget.
getXLen();
17836 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17837 if (EltBits <= XLen)
17838 return XLen - EltBits + 1;
17842 unsigned IntNo =
Op.getConstantOperandVal(1);
17846 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17847 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17848 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17849 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17850 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17851 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17852 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17853 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17854 case Intrinsic::riscv_masked_cmpxchg_i64:
17862 assert(Subtarget.hasStdExtA());
17877 switch (
Op.getOpcode()) {
17883 return !
Op.getValueType().isInteger();
17891 assert(Ld &&
"Unexpected null LoadSDNode");
17899 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17900 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17901 CNode->getOffset() != 0)
17909 auto *CNode = GetSupportedConstantPool(
Ptr);
17910 if (!CNode || CNode->getTargetFlags() != 0)
17913 return CNode->getConstVal();
17921 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17922 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17928 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17931 return CNodeLo->getConstVal();
17936 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17968 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17971 int64_t LoCounter =
MI.getOperand(2).getImm();
17972 int64_t HiCounter =
MI.getOperand(3).getImm();
17982 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17994 MI.eraseFromParent();
18002 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
18010 Register SrcReg =
MI.getOperand(2).getReg();
18030 MI.eraseFromParent();
18037 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
18038 "Unexpected instruction");
18044 Register DstReg =
MI.getOperand(0).getReg();
18067 MI.eraseFromParent();
18072 switch (
MI.getOpcode()) {
18075 case RISCV::Select_GPR_Using_CC_GPR:
18076 case RISCV::Select_GPR_Using_CC_Imm:
18077 case RISCV::Select_FPR16_Using_CC_GPR:
18078 case RISCV::Select_FPR16INX_Using_CC_GPR:
18079 case RISCV::Select_FPR32_Using_CC_GPR:
18080 case RISCV::Select_FPR32INX_Using_CC_GPR:
18081 case RISCV::Select_FPR64_Using_CC_GPR:
18082 case RISCV::Select_FPR64INX_Using_CC_GPR:
18083 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18089 unsigned RelOpcode,
unsigned EqOpcode,
18092 Register DstReg =
MI.getOperand(0).getReg();
18093 Register Src1Reg =
MI.getOperand(1).getReg();
18094 Register Src2Reg =
MI.getOperand(2).getReg();
18096 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18120 MI.eraseFromParent();
18171 F->insert(It, FirstMBB);
18172 F->insert(It, SecondMBB);
18173 F->insert(It, SinkMBB);
18222 First.eraseFromParent();
18261 if (
MI.getOperand(2).isReg())
18262 RHS =
MI.getOperand(2).getReg();
18267 SelectDests.
insert(
MI.getOperand(0).getReg());
18271 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
18272 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
18273 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
18274 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
18275 Next->getOperand(5).isKill()) {
18280 SequenceMBBI != E; ++SequenceMBBI) {
18281 if (SequenceMBBI->isDebugInstr())
18284 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
18285 !SequenceMBBI->getOperand(2).isReg() ||
18286 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
18287 SequenceMBBI->getOperand(3).getImm() !=
CC ||
18288 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
18289 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
18291 LastSelectPseudo = &*SequenceMBBI;
18293 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
18296 if (SequenceMBBI->hasUnmodeledSideEffects() ||
18297 SequenceMBBI->mayLoadOrStore() ||
18298 SequenceMBBI->usesCustomInsertionHook())
18301 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
18316 F->insert(
I, IfFalseMBB);
18317 F->insert(
I, TailMBB);
18321 TailMBB->
push_back(DebugInstr->removeFromParent());
18325 TailMBB->
splice(TailMBB->
end(), HeadMBB,
18335 if (
MI.getOperand(2).isImm())
18338 .
addImm(
MI.getOperand(2).getImm())
18350 auto SelectMBBI =
MI.getIterator();
18351 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
18352 auto InsertionPoint = TailMBB->
begin();
18353 while (SelectMBBI != SelectEnd) {
18354 auto Next = std::next(SelectMBBI);
18357 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
18358 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
18359 .
addReg(SelectMBBI->getOperand(4).getReg())
18361 .
addReg(SelectMBBI->getOperand(5).getReg())
18376 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
18377 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
18379 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
18380 assert(Masked &&
"Could not find masked instruction for LMUL and SEW pair");
18386 unsigned CVTXOpc) {
18392 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18405 .
add(
MI.getOperand(1))
18406 .
add(
MI.getOperand(2))
18407 .
add(
MI.getOperand(3))
18409 .
add(
MI.getOperand(4))
18410 .
add(
MI.getOperand(5))
18411 .
add(
MI.getOperand(6))
18426 .
add(
MI.getOperand(0))
18427 .
add(
MI.getOperand(1))
18429 .
add(
MI.getOperand(3))
18431 .
add(
MI.getOperand(4))
18432 .
add(
MI.getOperand(5))
18433 .
add(
MI.getOperand(6))
18443 MI.eraseFromParent();
18449 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
18451 switch (
MI.getOpcode()) {
18454 case RISCV::PseudoFROUND_H:
18455 CmpOpc = RISCV::FLT_H;
18456 F2IOpc = RISCV::FCVT_W_H;
18457 I2FOpc = RISCV::FCVT_H_W;
18458 FSGNJOpc = RISCV::FSGNJ_H;
18459 FSGNJXOpc = RISCV::FSGNJX_H;
18460 RC = &RISCV::FPR16RegClass;
18462 case RISCV::PseudoFROUND_H_INX:
18463 CmpOpc = RISCV::FLT_H_INX;
18464 F2IOpc = RISCV::FCVT_W_H_INX;
18465 I2FOpc = RISCV::FCVT_H_W_INX;
18466 FSGNJOpc = RISCV::FSGNJ_H_INX;
18467 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18468 RC = &RISCV::GPRF16RegClass;
18470 case RISCV::PseudoFROUND_S:
18471 CmpOpc = RISCV::FLT_S;
18472 F2IOpc = RISCV::FCVT_W_S;
18473 I2FOpc = RISCV::FCVT_S_W;
18474 FSGNJOpc = RISCV::FSGNJ_S;
18475 FSGNJXOpc = RISCV::FSGNJX_S;
18476 RC = &RISCV::FPR32RegClass;
18478 case RISCV::PseudoFROUND_S_INX:
18479 CmpOpc = RISCV::FLT_S_INX;
18480 F2IOpc = RISCV::FCVT_W_S_INX;
18481 I2FOpc = RISCV::FCVT_S_W_INX;
18482 FSGNJOpc = RISCV::FSGNJ_S_INX;
18483 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18484 RC = &RISCV::GPRF32RegClass;
18486 case RISCV::PseudoFROUND_D:
18488 CmpOpc = RISCV::FLT_D;
18489 F2IOpc = RISCV::FCVT_L_D;
18490 I2FOpc = RISCV::FCVT_D_L;
18491 FSGNJOpc = RISCV::FSGNJ_D;
18492 FSGNJXOpc = RISCV::FSGNJX_D;
18493 RC = &RISCV::FPR64RegClass;
18495 case RISCV::PseudoFROUND_D_INX:
18497 CmpOpc = RISCV::FLT_D_INX;
18498 F2IOpc = RISCV::FCVT_L_D_INX;
18499 I2FOpc = RISCV::FCVT_D_L_INX;
18500 FSGNJOpc = RISCV::FSGNJ_D_INX;
18501 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18502 RC = &RISCV::GPRRegClass;
18514 F->insert(
I, CvtMBB);
18515 F->insert(
I, DoneMBB);
18526 Register DstReg =
MI.getOperand(0).getReg();
18527 Register SrcReg =
MI.getOperand(1).getReg();
18528 Register MaxReg =
MI.getOperand(2).getReg();
18529 int64_t FRM =
MI.getOperand(3).getImm();
18534 Register FabsReg =
MRI.createVirtualRegister(RC);
18538 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18553 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18575 MI.eraseFromParent();
18582 switch (
MI.getOpcode()) {
18585 case RISCV::ReadCounterWide:
18587 "ReadCounterWide is only to be used on riscv32");
18589 case RISCV::Select_GPR_Using_CC_GPR:
18590 case RISCV::Select_GPR_Using_CC_Imm:
18591 case RISCV::Select_FPR16_Using_CC_GPR:
18592 case RISCV::Select_FPR16INX_Using_CC_GPR:
18593 case RISCV::Select_FPR32_Using_CC_GPR:
18594 case RISCV::Select_FPR32INX_Using_CC_GPR:
18595 case RISCV::Select_FPR64_Using_CC_GPR:
18596 case RISCV::Select_FPR64INX_Using_CC_GPR:
18597 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18599 case RISCV::BuildPairF64Pseudo:
18601 case RISCV::SplitF64Pseudo:
18603 case RISCV::PseudoQuietFLE_H:
18605 case RISCV::PseudoQuietFLE_H_INX:
18606 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18607 case RISCV::PseudoQuietFLT_H:
18609 case RISCV::PseudoQuietFLT_H_INX:
18610 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18611 case RISCV::PseudoQuietFLE_S:
18613 case RISCV::PseudoQuietFLE_S_INX:
18614 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18615 case RISCV::PseudoQuietFLT_S:
18617 case RISCV::PseudoQuietFLT_S_INX:
18618 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18619 case RISCV::PseudoQuietFLE_D:
18621 case RISCV::PseudoQuietFLE_D_INX:
18622 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18623 case RISCV::PseudoQuietFLE_D_IN32X:
18626 case RISCV::PseudoQuietFLT_D:
18628 case RISCV::PseudoQuietFLT_D_INX:
18629 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18630 case RISCV::PseudoQuietFLT_D_IN32X:
18634 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18636 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18638 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18640 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18642 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18644 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18646 case RISCV::PseudoFROUND_H:
18647 case RISCV::PseudoFROUND_H_INX:
18648 case RISCV::PseudoFROUND_S:
18649 case RISCV::PseudoFROUND_S_INX:
18650 case RISCV::PseudoFROUND_D:
18651 case RISCV::PseudoFROUND_D_INX:
18652 case RISCV::PseudoFROUND_D_IN32X:
18654 case TargetOpcode::STATEPOINT:
18660 MI.addOperand(*
MI.getMF(),
18666 case TargetOpcode::STACKMAP:
18667 case TargetOpcode::PATCHPOINT:
18670 "supported on 64-bit targets");
18688 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18720 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18721 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18724 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18725 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18728 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18729 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18733 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18734 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18735 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18737 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18738 RISCV::V20M2, RISCV::V22M2};
18746 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18747 RISCV::X13, RISCV::X14, RISCV::X15,
18748 RISCV::X16, RISCV::X17};
18750 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18751 RISCV::X13, RISCV::X14, RISCV::X15};
18762 static const MCPhysReg FastCCIGPRs[] = {
18763 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
18764 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
18765 RISCV::X29, RISCV::X30, RISCV::X31};
18768 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18769 RISCV::X13, RISCV::X14, RISCV::X15,
18784 unsigned XLenInBytes = XLen / 8;
18797 Align StackAlign(XLenInBytes);
18798 if (!
EABI || XLen != 32)
18830 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18831 assert(XLen == 32 || XLen == 64);
18832 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18836 if (ArgFlags.
isNest()) {
18837 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18845 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18850 bool UseGPRForF16_F32 =
true;
18853 bool UseGPRForF64 =
true;
18865 UseGPRForF16_F32 = !IsFixed;
18869 UseGPRForF16_F32 = !IsFixed;
18870 UseGPRForF64 = !IsFixed;
18876 UseGPRForF16_F32 =
true;
18877 UseGPRForF64 =
true;
18884 if (UseGPRForF16_F32 &&
18885 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18888 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18905 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18907 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18911 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18920 "PendingLocs and PendingArgFlags out of sync");
18924 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18925 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18973 PendingLocs.
size() <= 2) {
18974 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18979 PendingLocs.
clear();
18980 PendingArgFlags.
clear();
18982 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18988 unsigned StoreSizeBytes = XLen / 8;
18991 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18993 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18995 else if (ValVT == MVT::f64 && !UseGPRForF64)
19031 if (!PendingLocs.
empty()) {
19033 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
19035 for (
auto &It : PendingLocs) {
19037 It.convertToReg(Reg);
19042 PendingLocs.clear();
19043 PendingArgFlags.
clear();
19047 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
19049 "Expected an XLenVT or vector types at this stage");
19067template <
typename ArgTy>
19069 for (
const auto &ArgIdx :
enumerate(Args)) {
19070 MVT ArgVT = ArgIdx.value().VT;
19072 return ArgIdx.index();
19074 return std::nullopt;
19077void RISCVTargetLowering::analyzeInputArgs(
19080 RISCVCCAssignFn Fn)
const {
19081 unsigned NumArgs =
Ins.size();
19094 for (
unsigned i = 0; i != NumArgs; ++i) {
19098 Type *ArgTy =
nullptr;
19100 ArgTy = FType->getReturnType();
19101 else if (Ins[i].isOrigArg())
19102 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
19106 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
19108 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
19115void RISCVTargetLowering::analyzeOutputArgs(
19118 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
19119 unsigned NumArgs = Outs.
size();
19129 for (
unsigned i = 0; i != NumArgs; i++) {
19130 MVT ArgVT = Outs[i].VT;
19132 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
19136 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
19138 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
19192 if (In.isOrigArg()) {
19197 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
19198 (
BitWidth < 32 && In.Flags.isZExt())) {
19227 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
19273 ExtType,
DL, LocVT, Chain, FIN,
19290 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19303 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19313 unsigned ValNo,
MVT ValVT,
MVT LocVT,
19316 bool IsFixed,
bool IsRet,
Type *OrigTy,
19319 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19328 if (LocVT == MVT::f16 &&
19329 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
19331 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
19332 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
19333 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
19334 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
19335 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
19341 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19343 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
19344 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
19345 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
19346 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
19347 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19353 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19355 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
19356 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
19357 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
19358 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
19359 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19366 if ((LocVT == MVT::f16 &&
19367 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
19368 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19369 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
19370 Subtarget.hasStdExtZdinx())) {
19377 if (LocVT == MVT::f16) {
19383 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
19389 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
19397 if (AllocatedVReg) {
19432 if (ArgFlags.
isNest()) {
19434 "Attribute 'nest' is not supported in GHC calling convention");
19438 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
19439 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
19441 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19453 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19456 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
19457 RISCV::F18_F, RISCV::F19_F,
19458 RISCV::F20_F, RISCV::F21_F};
19459 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19465 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19468 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
19469 RISCV::F24_D, RISCV::F25_D,
19470 RISCV::F26_D, RISCV::F27_D};
19471 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19477 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19478 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
19498 switch (CallConv) {
19508 if (Subtarget.hasStdExtE())
19512 "(Zdinx/D) instruction set extensions");
19516 if (Func.hasFnAttribute(
"interrupt")) {
19517 if (!Func.arg_empty())
19519 "Functions with the interrupt attribute cannot have arguments!");
19524 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19526 "Function interrupt attribute argument not supported!");
19531 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19533 std::vector<SDValue> OutChains;
19542 analyzeInputArgs(MF, CCInfo, Ins,
false,
19546 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19567 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19568 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19570 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19572 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19601 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19606 if (VarArgsSaveSize == 0) {
19610 int VaArgOffset = -VarArgsSaveSize;
19618 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19619 VarArgsSaveSize += XLenInBytes;
19626 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19631 Chain,
DL, ArgValue, FIN,
19633 OutChains.push_back(Store);
19647 if (!OutChains.empty()) {
19648 OutChains.push_back(Chain);
19658bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19662 auto CalleeCC = CLI.CallConv;
19663 auto &Outs = CLI.Outs;
19665 auto CallerCC = Caller.getCallingConv();
19672 if (Caller.hasFnAttribute(
"interrupt"))
19687 for (
auto &VA : ArgLocs)
19693 auto IsCallerStructRet = Caller.hasStructRetAttr();
19694 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19695 if (IsCallerStructRet || IsCalleeStructRet)
19700 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19701 if (CalleeCC != CallerCC) {
19702 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19703 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19710 for (
auto &Arg : Outs)
19711 if (Arg.Flags.isByVal())
19746 if (Subtarget.hasStdExtE())
19750 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19756 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19762 "site marked musttail");
19769 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19771 if (!Flags.isByVal())
19775 unsigned Size = Flags.getByValSize();
19776 Align Alignment = Flags.getNonZeroByValAlign();
19783 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19797 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19800 SDValue ArgValue = OutVals[OutIdx];
19820 if (!StackPtr.getNode())
19831 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19849 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19850 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19856 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19857 SDValue PartValue = OutVals[OutIdx + 1];
19858 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19870 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19872 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19874 for (
const auto &Part : Parts) {
19875 SDValue PartValue = Part.first;
19876 SDValue PartOffset = Part.second;
19883 ArgValue = SpillSlot;
19889 if (Flags.isByVal())
19890 ArgValue = ByValArgs[j++];
19897 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19898 "for passing parameters");
19901 if (!StackPtr.getNode())
19914 if (!MemOpChains.
empty())
19920 for (
auto &Reg : RegsToPass) {
19921 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19928 validateCCReservedRegs(RegsToPass, MF);
19933 "Return address register required, but has been reserved."});
19952 for (
auto &Reg : RegsToPass)
19958 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19959 assert(Mask &&
"Missing call preserved mask for calling convention");
19968 "Unexpected CFI type for a direct call");
19998 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19999 auto &VA = RVLocs[i];
20007 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
20008 assert(VA.needsCustom());
20029 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
20033 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20034 MVT VT = Outs[i].VT;
20038 ArgFlags, CCInfo,
true,
true,
20039 nullptr, *
this, Dispatcher))
20071 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
20072 SDValue Val = OutVals[OutIdx];
20081 DAG.
getVTList(MVT::i32, MVT::i32), Val);
20085 Register RegHi = RVLocs[++i].getLocReg();
20091 "Return value register required, but has been reserved."});
20107 "Return value register required, but has been reserved."});
20129 if (Func.hasFnAttribute(
"interrupt")) {
20130 if (!Func.getReturnType()->isVoidTy())
20132 "Functions with the interrupt attribute must have void return type!");
20138 if (Kind ==
"supervisor")
20144 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
20147void RISCVTargetLowering::validateCCReservedRegs(
20148 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
20157 F,
"Argument register required, but has been reserved."});
20163 if (
N->getNumValues() != 1)
20165 if (!
N->hasNUsesOfValue(1, 0))
20168 SDNode *Copy = *
N->use_begin();
20182 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20186 bool HasRet =
false;
20187 for (
SDNode *Node : Copy->uses()) {
20195 Chain = Copy->getOperand(0);
20204#define NODE_NAME_CASE(NODE) \
20205 case RISCVISD::NODE: \
20206 return "RISCVISD::" #NODE;
20464#undef NODE_NAME_CASE
20471 if (Constraint.
size() == 1) {
20472 switch (Constraint[0]) {
20488 if (Constraint ==
"vr" || Constraint ==
"vm")
20494std::pair<unsigned, const TargetRegisterClass *>
20500 if (Constraint.
size() == 1) {
20501 switch (Constraint[0]) {
20506 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20507 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20508 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20509 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20510 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20511 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20512 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20514 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20515 return std::make_pair(0U, &RISCV::FPR16RegClass);
20516 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20517 return std::make_pair(0U, &RISCV::FPR32RegClass);
20518 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20519 return std::make_pair(0U, &RISCV::FPR64RegClass);
20524 }
else if (Constraint ==
"vr") {
20525 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
20526 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20528 return std::make_pair(0U, RC);
20530 }
else if (Constraint ==
"vm") {
20531 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20532 return std::make_pair(0U, &RISCV::VMV0RegClass);
20540 .
Case(
"{zero}", RISCV::X0)
20541 .
Case(
"{ra}", RISCV::X1)
20542 .
Case(
"{sp}", RISCV::X2)
20543 .
Case(
"{gp}", RISCV::X3)
20544 .
Case(
"{tp}", RISCV::X4)
20545 .
Case(
"{t0}", RISCV::X5)
20546 .
Case(
"{t1}", RISCV::X6)
20547 .
Case(
"{t2}", RISCV::X7)
20548 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20549 .
Case(
"{s1}", RISCV::X9)
20550 .
Case(
"{a0}", RISCV::X10)
20551 .
Case(
"{a1}", RISCV::X11)
20552 .
Case(
"{a2}", RISCV::X12)
20553 .
Case(
"{a3}", RISCV::X13)
20554 .
Case(
"{a4}", RISCV::X14)
20555 .
Case(
"{a5}", RISCV::X15)
20556 .
Case(
"{a6}", RISCV::X16)
20557 .
Case(
"{a7}", RISCV::X17)
20558 .
Case(
"{s2}", RISCV::X18)
20559 .
Case(
"{s3}", RISCV::X19)
20560 .
Case(
"{s4}", RISCV::X20)
20561 .
Case(
"{s5}", RISCV::X21)
20562 .
Case(
"{s6}", RISCV::X22)
20563 .
Case(
"{s7}", RISCV::X23)
20564 .
Case(
"{s8}", RISCV::X24)
20565 .
Case(
"{s9}", RISCV::X25)
20566 .
Case(
"{s10}", RISCV::X26)
20567 .
Case(
"{s11}", RISCV::X27)
20568 .
Case(
"{t3}", RISCV::X28)
20569 .
Case(
"{t4}", RISCV::X29)
20570 .
Case(
"{t5}", RISCV::X30)
20571 .
Case(
"{t6}", RISCV::X31)
20573 if (XRegFromAlias != RISCV::NoRegister)
20574 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20583 if (Subtarget.hasStdExtF()) {
20585 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20586 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20587 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20588 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20589 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20590 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20591 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20592 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20593 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20594 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20595 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20596 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20597 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20598 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20599 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20600 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20601 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20602 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20603 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20604 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20605 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20606 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20607 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20608 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20609 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20610 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20611 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20612 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20613 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20614 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20615 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20616 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20618 if (FReg != RISCV::NoRegister) {
20619 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20620 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20621 unsigned RegNo = FReg - RISCV::F0_F;
20622 unsigned DReg = RISCV::F0_D + RegNo;
20623 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20625 if (VT == MVT::f32 || VT == MVT::Other)
20626 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20627 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20628 unsigned RegNo = FReg - RISCV::F0_F;
20629 unsigned HReg = RISCV::F0_H + RegNo;
20630 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20637 .
Case(
"{v0}", RISCV::V0)
20638 .
Case(
"{v1}", RISCV::V1)
20639 .
Case(
"{v2}", RISCV::V2)
20640 .
Case(
"{v3}", RISCV::V3)
20641 .
Case(
"{v4}", RISCV::V4)
20642 .
Case(
"{v5}", RISCV::V5)
20643 .
Case(
"{v6}", RISCV::V6)
20644 .
Case(
"{v7}", RISCV::V7)
20645 .
Case(
"{v8}", RISCV::V8)
20646 .
Case(
"{v9}", RISCV::V9)
20647 .
Case(
"{v10}", RISCV::V10)
20648 .
Case(
"{v11}", RISCV::V11)
20649 .
Case(
"{v12}", RISCV::V12)
20650 .
Case(
"{v13}", RISCV::V13)
20651 .
Case(
"{v14}", RISCV::V14)
20652 .
Case(
"{v15}", RISCV::V15)
20653 .
Case(
"{v16}", RISCV::V16)
20654 .
Case(
"{v17}", RISCV::V17)
20655 .
Case(
"{v18}", RISCV::V18)
20656 .
Case(
"{v19}", RISCV::V19)
20657 .
Case(
"{v20}", RISCV::V20)
20658 .
Case(
"{v21}", RISCV::V21)
20659 .
Case(
"{v22}", RISCV::V22)
20660 .
Case(
"{v23}", RISCV::V23)
20661 .
Case(
"{v24}", RISCV::V24)
20662 .
Case(
"{v25}", RISCV::V25)
20663 .
Case(
"{v26}", RISCV::V26)
20664 .
Case(
"{v27}", RISCV::V27)
20665 .
Case(
"{v28}", RISCV::V28)
20666 .
Case(
"{v29}", RISCV::V29)
20667 .
Case(
"{v30}", RISCV::V30)
20668 .
Case(
"{v31}", RISCV::V31)
20670 if (VReg != RISCV::NoRegister) {
20671 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20672 return std::make_pair(VReg, &RISCV::VMRegClass);
20673 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20674 return std::make_pair(VReg, &RISCV::VRRegClass);
20675 for (
const auto *RC :
20676 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20677 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20678 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20679 return std::make_pair(VReg, RC);
20685 std::pair<Register, const TargetRegisterClass *> Res =
20691 if (Res.second == &RISCV::GPRF16RegClass ||
20692 Res.second == &RISCV::GPRF32RegClass ||
20693 Res.second == &RISCV::GPRPairRegClass)
20694 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20702 if (ConstraintCode.
size() == 1) {
20703 switch (ConstraintCode[0]) {
20718 if (Constraint.
size() == 1) {
20719 switch (Constraint[0]) {
20722 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20724 if (isInt<12>(CVal))
20737 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20739 if (isUInt<5>(CVal))
20757 if (Subtarget.hasStdExtZtso()) {
20773 if (Subtarget.hasStdExtZtso()) {
20781 if (Subtarget.enableSeqCstTrailingFence() && isa<StoreInst>(Inst) &&
20798 if (Subtarget.hasForcedAtomics())
20803 if (Subtarget.hasStdExtZacas() &&
20804 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20810 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20823 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20825 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20827 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20829 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20831 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20833 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20835 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20837 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20846 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20848 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20850 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20852 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20854 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20856 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20858 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20860 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20886 unsigned XLen = Subtarget.
getXLen();
20910 unsigned ValWidth =
20915 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20918 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20930 if (Subtarget.hasForcedAtomics())
20934 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20943 unsigned XLen = Subtarget.
getXLen();
20945 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20950 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20956 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20963 EVT DataVT)
const {
20979 return Subtarget.hasStdExtZfhmin();
20981 return Subtarget.hasStdExtF();
20983 return Subtarget.hasStdExtD();
21015 "RVVBitsPerBlock changed, audit needed");
21024 if (!Subtarget.hasVendorXTHeadMemIdx())
21030 Base =
Op->getOperand(0);
21032 int64_t RHSC =
RHS->getSExtValue();
21038 bool isLegalIndexedOffset =
false;
21039 for (
unsigned i = 0; i < 4; i++)
21040 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
21041 isLegalIndexedOffset =
true;
21045 if (!isLegalIndexedOffset)
21062 VT = LD->getMemoryVT();
21063 Ptr = LD->getBasePtr();
21064 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21065 VT = ST->getMemoryVT();
21066 Ptr = ST->getBasePtr();
21082 if (Subtarget.hasVendorXCVmem()) {
21087 Base = LS->getBasePtr();
21091 if (
Base ==
Op->getOperand(0))
21093 else if (
Base ==
Op->getOperand(1))
21105 VT = LD->getMemoryVT();
21106 Ptr = LD->getBasePtr();
21107 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21108 VT = ST->getMemoryVT();
21109 Ptr = ST->getBasePtr();
21152 const Constant *PersonalityFn)
const {
21157 const Constant *PersonalityFn)
const {
21181 const bool HasZmmul = Subtarget.hasStdExtZmmul();
21190 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
21192 const APInt &Imm = ConstNode->getAPIntValue();
21193 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
21194 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
21198 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
21199 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
21200 (Imm - 8).isPowerOf2()))
21205 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
21206 ConstNode->hasOneUse()) {
21207 APInt ImmS = Imm.ashr(Imm.countr_zero());
21208 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
21209 (1 - ImmS).isPowerOf2())
21233 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
21242 unsigned *
Fast)
const {
21245 *
Fast = Subtarget.enableUnalignedScalarMem();
21246 return Subtarget.enableUnalignedScalarMem();
21262 *
Fast = Subtarget.enableUnalignedVectorMem();
21263 return Subtarget.enableUnalignedVectorMem();
21272 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
21284 if (
Op.size() < MinVLenInBytes)
21294 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
21298 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
21300 if (
Op.isFixedDstAlign())
21301 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
21303 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
21311 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
21312 bool IsABIRegCopy =
CC.has_value();
21314 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21315 PartVT == MVT::f32) {
21333 if (PartVTBitSize % ValueVTBitSize == 0) {
21334 assert(PartVTBitSize >= ValueVTBitSize);
21341 if (ValueEltVT != PartEltVT) {
21342 if (PartVTBitSize > ValueVTBitSize) {
21344 assert(Count != 0 &&
"The number of element should not be zero.");
21345 EVT SameEltTypeVT =
21366 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
21367 bool IsABIRegCopy =
CC.has_value();
21368 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21369 PartVT == MVT::f32) {
21386 if (PartVTBitSize % ValueVTBitSize == 0) {
21387 assert(PartVTBitSize >= ValueVTBitSize);
21388 EVT SameEltTypeVT = ValueVT;
21395 if (ValueEltVT != PartEltVT) {
21397 assert(Count != 0 &&
"The number of element should not be zero.");
21414 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21421 unsigned Opc =
N->getOpcode();
21451 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21465 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21470 if (FVTy->getNumElements() < 2)
21485 return Factor * LMUL <= 8;
21489 Align Alignment)
const {
21501 if (!Subtarget.enableUnalignedVectorMem() &&
21509 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21510 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21511 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21512 Intrinsic::riscv_seg8_load};
21531 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21541 {VTy, LI->getPointerOperandType(), XLenTy});
21543 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21548 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21550 Shuffles[i]->replaceAllUsesWith(SubVec);
21557 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21558 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21559 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21560 Intrinsic::riscv_seg8_store};
21580 unsigned Factor)
const {
21582 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21585 ShuffleVTy->getNumElements() / Factor);
21587 SI->getPointerAddressSpace(),
21588 SI->getDataLayout()))
21595 {VTy, SI->getPointerOperandType(), XLenTy});
21600 for (
unsigned i = 0; i < Factor; i++) {
21609 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21610 Ops.
append({SI->getPointerOperand(), VL});
21626 unsigned Factor = 2;
21641 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21644 {ResVTy, LI->getPointerOperandType(), XLenTy});
21645 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21648 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21649 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21650 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21651 Intrinsic::riscv_vlseg8};
21673 if (
II->getIntrinsicID() != Intrinsic::vector_interleave2)
21676 unsigned Factor = 2;
21679 VectorType *InVTy = cast<VectorType>(
II->getOperand(0)->getType());
21682 SI->getPointerAddressSpace(),
21683 SI->getDataLayout()))
21690 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21693 {InVTy, SI->getPointerOperandType(), XLenTy});
21694 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21697 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21698 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21699 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21700 Intrinsic::riscv_vsseg8};
21707 Builder.
CreateCall(VssegNFunc, {
II->getOperand(0),
II->getOperand(1),
21708 SI->getPointerOperand(), VL});
21718 "Invalid call instruction for a KCFI check");
21720 MBBI->getOpcode()));
21723 Target.setIsRenamable(
false);
21731#define GET_REGISTER_MATCHER
21732#include "RISCVGenAsmMatcher.inc"
21738 if (Reg == RISCV::NoRegister)
21740 if (Reg == RISCV::NoRegister)
21752 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21754 if (NontemporalInfo ==
nullptr)
21762 int NontemporalLevel = 5;
21763 const MDNode *RISCVNontemporalInfo =
21764 I.getMetadata(
"riscv-nontemporal-domain");
21765 if (RISCVNontemporalInfo !=
nullptr)
21768 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21772 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21773 "RISC-V target doesn't support this non-temporal domain.");
21775 NontemporalLevel -= 2;
21777 if (NontemporalLevel & 0b1)
21779 if (NontemporalLevel & 0b10)
21792 return TargetFlags;
21802 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21805 return Subtarget.hasStdExtZbb() &&
21818 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21819 Op == Instruction::And ||
Op == Instruction::Or ||
21820 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21821 Op == Instruction::ShuffleVector ||
Op == Instruction::Load ||
21822 Op == Instruction::Freeze ||
Op == Instruction::Store)
21830 !isa<ReturnInst>(&Inst))
21833 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21834 if (AI->getAllocatedType()->isScalableTy())
21842RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21850 if (!Subtarget.hasShortForwardBranchOpt())
21852 EVT VT =
N->getValueType(0);
21853 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21857 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21862bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21863 EVT VT,
const APInt &AndMask)
const {
21864 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21865 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21869unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21874template <
typename Arg>
21875void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21878 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21880 auto It = ArgList.
begin();
21881 MVT FirstArgRegType = It->VT;
21884 if (It == ArgList.
end() || It->Flags.isSplit())
21896 for (; It != ArgList.
end(); ++It)
21897 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21903 if (isHomogeneousScalableVectorType(ArgList)) {
21905 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21908 bool FirstVMaskAssigned =
false;
21909 for (
const auto &OutArg : ArgList) {
21910 MVT RegisterVT = OutArg.VT;
21920 RVVArgInfos.push_back({1, RegisterVT,
true});
21921 FirstVMaskAssigned =
true;
21925 RVVArgInfos.push_back({1, RegisterVT,
false});
21932void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21937 bool FirstVMaskAssigned =
false;
21938 for (
Type *Ty : TypeList) {
21948 RVVArgInfos.push_back(
21954 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21969 if (!FirstVMaskAssigned &&
21971 RVVArgInfos.push_back({1, RegisterVT,
true});
21972 FirstVMaskAssigned =
true;
21976 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21982void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21983 unsigned StartReg) {
21984 assert((StartReg % LMul) == 0 &&
21985 "Start register number should be multiple of lmul");
22004 for (
unsigned i = 0; i < NF; ++i)
22006 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
22008 AllocatedPhysRegs.push_back(
MCPhysReg());
22014void RVVArgDispatcher::compute() {
22016 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
22019 AllocatedPhysRegs.push_back(RISCV::V0);
22025 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
22026 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
22027 StartReg += RegsNeeded) {
22028 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
22029 if ((AssignedMap & Map) == 0) {
22030 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
22031 AssignedMap |=
Map;
22036 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
22039 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
22040 allocate(RVVArgInfos[i]);
22044 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
22045 return AllocatedPhysRegs[CurIdx++];
22052 if (Subtarget.hasStdExtZicfilp()) {
22064#define GET_RISCVVIntrinsicsTable_IMPL
22065#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static bool IsSelect(MachineInstr &MI)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool isTargetAndroid() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SW_GUARDED_BRIND
Software guarded BRIND node.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)