38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
253 if (!Subtarget.hasVendorXCValu())
257 if (!Subtarget.hasVendorXCValu())
276 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
288 if (!Subtarget.hasStdExtZbb())
292 if (Subtarget.hasStdExtZbb()) {
299 if (!Subtarget.hasStdExtZmmul()) {
303 }
else if (Subtarget.
is64Bit()) {
313 if (!Subtarget.hasStdExtM()) {
319 }
else if (Subtarget.
is64Bit()) {
322 {MVT::i8, MVT::i16, MVT::i32},
Custom);
339 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
342 }
else if (Subtarget.hasVendorXTHeadBb()) {
346 }
else if (Subtarget.hasVendorXCVbitmanip()) {
357 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
358 Subtarget.hasVendorXTHeadBb())
363 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
364 Subtarget.hasVendorXTHeadBb())
369 if (Subtarget.hasVendorXCVbitmanip()) {
377 if (Subtarget.hasStdExtZbb()) {
390 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
396 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
397 Subtarget.hasVendorXCVbitmanip()) {
404 if (!Subtarget.hasStdExtZbb())
416 !Subtarget.hasShortForwardBranchOpt())
420 if (Subtarget.hasShortForwardBranchOpt())
423 if (!Subtarget.hasVendorXTHeadCondMov()) {
429 static const unsigned FPLegalNodeTypes[] = {
442 static const unsigned FPOpToExpand[] = {
446 static const unsigned FPRndMode[] = {
453 static const unsigned ZfhminZfbfminPromoteOps[] = {
464 if (Subtarget.hasStdExtZfbfmin()) {
547 if (Subtarget.hasStdExtZfa()) {
564 if (Subtarget.hasStdExtZfa()) {
641 if (Subtarget.hasStdExtZicbop()) {
645 if (Subtarget.hasStdExtA()) {
647 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
651 }
else if (Subtarget.hasForcedAtomics()) {
677 {MVT::i8, MVT::i16},
Custom);
688 static const unsigned IntegerVPOps[] = {
689 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
690 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
691 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
692 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
693 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
694 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
695 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
696 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
697 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
698 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
699 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
700 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
701 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
702 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
703 ISD::EXPERIMENTAL_VP_SPLAT};
705 static const unsigned FloatingPointVPOps[] = {
706 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
707 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
708 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
709 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
710 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
711 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
712 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
713 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
714 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
715 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
716 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
717 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
718 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
719 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
721 static const unsigned IntegerVecReduceOps[] = {
726 static const unsigned FloatingPointVecReduceOps[] = {
739 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
740 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
741 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
745 for (
MVT VT : BoolVecVTs) {
775 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
799 ISD::VP_TRUNCATE, ISD::VP_SETCC},
815 for (
MVT VT : IntVecVTs) {
826 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
874 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
875 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
899 if (Subtarget.hasStdExtZvkb()) {
907 if (Subtarget.hasStdExtZvbb()) {
911 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
917 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
926 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
946 static const unsigned ZvfhminPromoteOps[] = {
956 static const unsigned ZvfhminPromoteVPOps[] = {
957 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
958 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
959 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
960 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
961 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
962 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
963 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
964 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
965 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
968 const auto SetCommonVFPActions = [&](
MVT VT) {
1013 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1014 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1045 const auto SetCommonVFPExtLoadTruncStoreActions =
1047 for (
auto SmallVT : SmallerVTs) {
1054 for (
MVT VT : F16VecVTs) {
1057 SetCommonVFPActions(VT);
1060 for (
MVT VT : F16VecVTs) {
1071 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1076 if (Subtarget.hasStdExtZfhmin())
1082 if (VT == MVT::nxv32f16) {
1096 for (
MVT VT : BF16VecVTs) {
1107 if (Subtarget.hasStdExtZfbfmin())
1117 for (
MVT VT : F32VecVTs) {
1120 SetCommonVFPActions(VT);
1121 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1126 for (
MVT VT : F64VecVTs) {
1129 SetCommonVFPActions(VT);
1130 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1131 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1137 if (!useRVVForFixedLengthVectorVT(VT))
1180 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1207 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1229 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1230 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1267 if (Subtarget.hasStdExtZvkb())
1270 if (Subtarget.hasStdExtZvbb()) {
1292 if (!useRVVForFixedLengthVectorVT(VT))
1322 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1325 if (Subtarget.hasStdExtZfhmin()) {
1363 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1364 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1410 if (Subtarget.hasStdExtA()) {
1416 if (Subtarget.hasForcedAtomics()) {
1426 if (Subtarget.hasVendorXTHeadMemIdx()) {
1442 if (Subtarget.hasVendorXCVmem()) {
1452 if (Subtarget.hasVendorXCValu()) {
1478 if (Subtarget.hasStdExtZbb())
1481 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1482 Subtarget.hasStdExtV())
1485 if (Subtarget.hasStdExtZbkb())
1497 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1500 if (Subtarget.hasVendorXTHeadMemPair())
1528MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1533bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1535 bool IsScalable)
const {
1542 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1567 unsigned Intrinsic)
const {
1568 auto &
DL =
I.getDataLayout();
1570 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1571 bool IsUnitStrided,
bool UsePtrVal =
false) {
1576 Info.ptrVal =
I.getArgOperand(PtrOp);
1578 Info.fallbackAddressSpace =
1579 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1583 MemTy =
I.getArgOperand(0)->getType();
1586 MemTy =
I.getType();
1601 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1605 switch (Intrinsic) {
1608 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1609 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1610 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1611 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1612 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1613 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1614 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1615 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1616 case Intrinsic::riscv_masked_cmpxchg_i32:
1618 Info.memVT = MVT::i32;
1619 Info.ptrVal =
I.getArgOperand(0);
1625 case Intrinsic::riscv_masked_strided_load:
1626 return SetRVVLoadStoreInfo( 1,
false,
1628 case Intrinsic::riscv_masked_strided_store:
1629 return SetRVVLoadStoreInfo( 1,
true,
1631 case Intrinsic::riscv_seg2_load:
1632 case Intrinsic::riscv_seg3_load:
1633 case Intrinsic::riscv_seg4_load:
1634 case Intrinsic::riscv_seg5_load:
1635 case Intrinsic::riscv_seg6_load:
1636 case Intrinsic::riscv_seg7_load:
1637 case Intrinsic::riscv_seg8_load:
1638 return SetRVVLoadStoreInfo( 0,
false,
1640 case Intrinsic::riscv_seg2_store:
1641 case Intrinsic::riscv_seg3_store:
1642 case Intrinsic::riscv_seg4_store:
1643 case Intrinsic::riscv_seg5_store:
1644 case Intrinsic::riscv_seg6_store:
1645 case Intrinsic::riscv_seg7_store:
1646 case Intrinsic::riscv_seg8_store:
1648 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1651 case Intrinsic::riscv_vle:
1652 case Intrinsic::riscv_vle_mask:
1653 case Intrinsic::riscv_vleff:
1654 case Intrinsic::riscv_vleff_mask:
1655 return SetRVVLoadStoreInfo( 1,
1659 case Intrinsic::riscv_vse:
1660 case Intrinsic::riscv_vse_mask:
1661 return SetRVVLoadStoreInfo( 1,
1665 case Intrinsic::riscv_vlse:
1666 case Intrinsic::riscv_vlse_mask:
1667 case Intrinsic::riscv_vloxei:
1668 case Intrinsic::riscv_vloxei_mask:
1669 case Intrinsic::riscv_vluxei:
1670 case Intrinsic::riscv_vluxei_mask:
1671 return SetRVVLoadStoreInfo( 1,
1674 case Intrinsic::riscv_vsse:
1675 case Intrinsic::riscv_vsse_mask:
1676 case Intrinsic::riscv_vsoxei:
1677 case Intrinsic::riscv_vsoxei_mask:
1678 case Intrinsic::riscv_vsuxei:
1679 case Intrinsic::riscv_vsuxei_mask:
1680 return SetRVVLoadStoreInfo( 1,
1683 case Intrinsic::riscv_vlseg2:
1684 case Intrinsic::riscv_vlseg3:
1685 case Intrinsic::riscv_vlseg4:
1686 case Intrinsic::riscv_vlseg5:
1687 case Intrinsic::riscv_vlseg6:
1688 case Intrinsic::riscv_vlseg7:
1689 case Intrinsic::riscv_vlseg8:
1690 case Intrinsic::riscv_vlseg2ff:
1691 case Intrinsic::riscv_vlseg3ff:
1692 case Intrinsic::riscv_vlseg4ff:
1693 case Intrinsic::riscv_vlseg5ff:
1694 case Intrinsic::riscv_vlseg6ff:
1695 case Intrinsic::riscv_vlseg7ff:
1696 case Intrinsic::riscv_vlseg8ff:
1697 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1700 case Intrinsic::riscv_vlseg2_mask:
1701 case Intrinsic::riscv_vlseg3_mask:
1702 case Intrinsic::riscv_vlseg4_mask:
1703 case Intrinsic::riscv_vlseg5_mask:
1704 case Intrinsic::riscv_vlseg6_mask:
1705 case Intrinsic::riscv_vlseg7_mask:
1706 case Intrinsic::riscv_vlseg8_mask:
1707 case Intrinsic::riscv_vlseg2ff_mask:
1708 case Intrinsic::riscv_vlseg3ff_mask:
1709 case Intrinsic::riscv_vlseg4ff_mask:
1710 case Intrinsic::riscv_vlseg5ff_mask:
1711 case Intrinsic::riscv_vlseg6ff_mask:
1712 case Intrinsic::riscv_vlseg7ff_mask:
1713 case Intrinsic::riscv_vlseg8ff_mask:
1714 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1717 case Intrinsic::riscv_vlsseg2:
1718 case Intrinsic::riscv_vlsseg3:
1719 case Intrinsic::riscv_vlsseg4:
1720 case Intrinsic::riscv_vlsseg5:
1721 case Intrinsic::riscv_vlsseg6:
1722 case Intrinsic::riscv_vlsseg7:
1723 case Intrinsic::riscv_vlsseg8:
1724 case Intrinsic::riscv_vloxseg2:
1725 case Intrinsic::riscv_vloxseg3:
1726 case Intrinsic::riscv_vloxseg4:
1727 case Intrinsic::riscv_vloxseg5:
1728 case Intrinsic::riscv_vloxseg6:
1729 case Intrinsic::riscv_vloxseg7:
1730 case Intrinsic::riscv_vloxseg8:
1731 case Intrinsic::riscv_vluxseg2:
1732 case Intrinsic::riscv_vluxseg3:
1733 case Intrinsic::riscv_vluxseg4:
1734 case Intrinsic::riscv_vluxseg5:
1735 case Intrinsic::riscv_vluxseg6:
1736 case Intrinsic::riscv_vluxseg7:
1737 case Intrinsic::riscv_vluxseg8:
1738 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1741 case Intrinsic::riscv_vlsseg2_mask:
1742 case Intrinsic::riscv_vlsseg3_mask:
1743 case Intrinsic::riscv_vlsseg4_mask:
1744 case Intrinsic::riscv_vlsseg5_mask:
1745 case Intrinsic::riscv_vlsseg6_mask:
1746 case Intrinsic::riscv_vlsseg7_mask:
1747 case Intrinsic::riscv_vlsseg8_mask:
1748 case Intrinsic::riscv_vloxseg2_mask:
1749 case Intrinsic::riscv_vloxseg3_mask:
1750 case Intrinsic::riscv_vloxseg4_mask:
1751 case Intrinsic::riscv_vloxseg5_mask:
1752 case Intrinsic::riscv_vloxseg6_mask:
1753 case Intrinsic::riscv_vloxseg7_mask:
1754 case Intrinsic::riscv_vloxseg8_mask:
1755 case Intrinsic::riscv_vluxseg2_mask:
1756 case Intrinsic::riscv_vluxseg3_mask:
1757 case Intrinsic::riscv_vluxseg4_mask:
1758 case Intrinsic::riscv_vluxseg5_mask:
1759 case Intrinsic::riscv_vluxseg6_mask:
1760 case Intrinsic::riscv_vluxseg7_mask:
1761 case Intrinsic::riscv_vluxseg8_mask:
1762 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1765 case Intrinsic::riscv_vsseg2:
1766 case Intrinsic::riscv_vsseg3:
1767 case Intrinsic::riscv_vsseg4:
1768 case Intrinsic::riscv_vsseg5:
1769 case Intrinsic::riscv_vsseg6:
1770 case Intrinsic::riscv_vsseg7:
1771 case Intrinsic::riscv_vsseg8:
1772 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1775 case Intrinsic::riscv_vsseg2_mask:
1776 case Intrinsic::riscv_vsseg3_mask:
1777 case Intrinsic::riscv_vsseg4_mask:
1778 case Intrinsic::riscv_vsseg5_mask:
1779 case Intrinsic::riscv_vsseg6_mask:
1780 case Intrinsic::riscv_vsseg7_mask:
1781 case Intrinsic::riscv_vsseg8_mask:
1782 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1785 case Intrinsic::riscv_vssseg2:
1786 case Intrinsic::riscv_vssseg3:
1787 case Intrinsic::riscv_vssseg4:
1788 case Intrinsic::riscv_vssseg5:
1789 case Intrinsic::riscv_vssseg6:
1790 case Intrinsic::riscv_vssseg7:
1791 case Intrinsic::riscv_vssseg8:
1792 case Intrinsic::riscv_vsoxseg2:
1793 case Intrinsic::riscv_vsoxseg3:
1794 case Intrinsic::riscv_vsoxseg4:
1795 case Intrinsic::riscv_vsoxseg5:
1796 case Intrinsic::riscv_vsoxseg6:
1797 case Intrinsic::riscv_vsoxseg7:
1798 case Intrinsic::riscv_vsoxseg8:
1799 case Intrinsic::riscv_vsuxseg2:
1800 case Intrinsic::riscv_vsuxseg3:
1801 case Intrinsic::riscv_vsuxseg4:
1802 case Intrinsic::riscv_vsuxseg5:
1803 case Intrinsic::riscv_vsuxseg6:
1804 case Intrinsic::riscv_vsuxseg7:
1805 case Intrinsic::riscv_vsuxseg8:
1806 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1809 case Intrinsic::riscv_vssseg2_mask:
1810 case Intrinsic::riscv_vssseg3_mask:
1811 case Intrinsic::riscv_vssseg4_mask:
1812 case Intrinsic::riscv_vssseg5_mask:
1813 case Intrinsic::riscv_vssseg6_mask:
1814 case Intrinsic::riscv_vssseg7_mask:
1815 case Intrinsic::riscv_vssseg8_mask:
1816 case Intrinsic::riscv_vsoxseg2_mask:
1817 case Intrinsic::riscv_vsoxseg3_mask:
1818 case Intrinsic::riscv_vsoxseg4_mask:
1819 case Intrinsic::riscv_vsoxseg5_mask:
1820 case Intrinsic::riscv_vsoxseg6_mask:
1821 case Intrinsic::riscv_vsoxseg7_mask:
1822 case Intrinsic::riscv_vsoxseg8_mask:
1823 case Intrinsic::riscv_vsuxseg2_mask:
1824 case Intrinsic::riscv_vsuxseg3_mask:
1825 case Intrinsic::riscv_vsuxseg4_mask:
1826 case Intrinsic::riscv_vsuxseg5_mask:
1827 case Intrinsic::riscv_vsuxseg6_mask:
1828 case Intrinsic::riscv_vsuxseg7_mask:
1829 case Intrinsic::riscv_vsuxseg8_mask:
1830 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1867 return isInt<12>(Imm);
1871 return isInt<12>(Imm);
1884 return (SrcBits == 64 && DestBits == 32);
1895 return (SrcBits == 64 && DestBits == 32);
1901 if (Subtarget.hasStdExtV() &&
1906 if (SrcBits == DestBits * 2) {
1917 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1918 EVT MemVT = LD->getMemoryVT();
1919 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1929 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1937 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1941 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1942 Subtarget.hasVendorXCVbitmanip();
1953 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1958 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1962 EVT VT =
Y.getValueType();
1968 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1969 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
1974 if (Subtarget.hasStdExtZbs())
1975 return X.getValueType().isScalarInteger();
1976 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1978 if (Subtarget.hasVendorXTHeadBs())
1979 return C !=
nullptr;
1981 return C &&
C->getAPIntValue().ule(10);
2001 if (BitSize > Subtarget.
getXLen())
2005 int64_t Val = Imm.getSExtValue();
2013 if (!Subtarget.enableUnalignedScalarMem())
2029 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
2036 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
2040 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2052 case Instruction::Add:
2053 case Instruction::Sub:
2054 case Instruction::Mul:
2055 case Instruction::And:
2056 case Instruction::Or:
2057 case Instruction::Xor:
2058 case Instruction::FAdd:
2059 case Instruction::FSub:
2060 case Instruction::FMul:
2061 case Instruction::FDiv:
2062 case Instruction::ICmp:
2063 case Instruction::FCmp:
2065 case Instruction::Shl:
2066 case Instruction::LShr:
2067 case Instruction::AShr:
2068 case Instruction::UDiv:
2069 case Instruction::SDiv:
2070 case Instruction::URem:
2071 case Instruction::SRem:
2072 case Instruction::Select:
2073 return Operand == 1;
2087 auto *
II = dyn_cast<IntrinsicInst>(
I);
2091 switch (
II->getIntrinsicID()) {
2092 case Intrinsic::fma:
2093 case Intrinsic::vp_fma:
2094 return Operand == 0 || Operand == 1;
2095 case Intrinsic::vp_shl:
2096 case Intrinsic::vp_lshr:
2097 case Intrinsic::vp_ashr:
2098 case Intrinsic::vp_udiv:
2099 case Intrinsic::vp_sdiv:
2100 case Intrinsic::vp_urem:
2101 case Intrinsic::vp_srem:
2102 case Intrinsic::ssub_sat:
2103 case Intrinsic::vp_ssub_sat:
2104 case Intrinsic::usub_sat:
2105 case Intrinsic::vp_usub_sat:
2106 return Operand == 1;
2108 case Intrinsic::vp_add:
2109 case Intrinsic::vp_mul:
2110 case Intrinsic::vp_and:
2111 case Intrinsic::vp_or:
2112 case Intrinsic::vp_xor:
2113 case Intrinsic::vp_fadd:
2114 case Intrinsic::vp_fmul:
2115 case Intrinsic::vp_icmp:
2116 case Intrinsic::vp_fcmp:
2117 case Intrinsic::smin:
2118 case Intrinsic::vp_smin:
2119 case Intrinsic::umin:
2120 case Intrinsic::vp_umin:
2121 case Intrinsic::smax:
2122 case Intrinsic::vp_smax:
2123 case Intrinsic::umax:
2124 case Intrinsic::vp_umax:
2125 case Intrinsic::sadd_sat:
2126 case Intrinsic::vp_sadd_sat:
2127 case Intrinsic::uadd_sat:
2128 case Intrinsic::vp_uadd_sat:
2130 case Intrinsic::vp_sub:
2131 case Intrinsic::vp_fsub:
2132 case Intrinsic::vp_fdiv:
2133 return Operand == 0 || Operand == 1;
2154 if (!Subtarget.sinkSplatOperands())
2157 for (
auto OpIdx :
enumerate(
I->operands())) {
2161 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2163 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2172 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2177 for (
Use &U :
Op->uses()) {
2229 if (!Subtarget.hasStdExtZfa())
2230 return std::make_pair(-1,
false);
2232 bool IsSupportedVT =
false;
2233 if (VT == MVT::f16) {
2234 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2235 }
else if (VT == MVT::f32) {
2236 IsSupportedVT =
true;
2237 }
else if (VT == MVT::f64) {
2238 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2239 IsSupportedVT =
true;
2243 return std::make_pair(-1,
false);
2246 if (
Index < 0 && Imm.isNegative())
2250 return std::make_pair(
Index,
false);
2254 bool ForCodeSize)
const {
2255 bool IsLegalVT =
false;
2258 else if (VT == MVT::f32)
2260 else if (VT == MVT::f64)
2262 else if (VT == MVT::bf16)
2263 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2275 return Imm.isZero();
2279 if (Imm.isNegZero())
2292 unsigned Index)
const {
2305 if (EltVT == MVT::i1)
2318 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2325 if ((ResElts * 2) != SrcElts)
2369 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2371 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2374 IntermediateVT = MVT::i64;
2377 RegisterVT = MVT::i64;
2392 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2398 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2411 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2412 int64_t
C = RHSC->getSExtValue();
2454 switch (KnownSize) {
2482 return RISCV::VRRegClassID;
2484 return RISCV::VRM2RegClassID;
2486 return RISCV::VRM4RegClassID;
2488 return RISCV::VRM8RegClassID;
2498 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2499 "Unexpected subreg numbering");
2500 return RISCV::sub_vrm1_0 +
Index;
2503 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2504 "Unexpected subreg numbering");
2505 return RISCV::sub_vrm2_0 +
Index;
2508 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2509 "Unexpected subreg numbering");
2510 return RISCV::sub_vrm4_0 +
Index;
2517 return RISCV::VRRegClassID;
2526std::pair<unsigned, unsigned>
2528 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2530 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2531 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2532 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2533 "Register classes not ordered");
2542 unsigned SubRegIdx = RISCV::NoSubRegister;
2543 for (
const unsigned RCID :
2544 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2545 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2549 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2554 return {SubRegIdx, InsertExtractIdx};
2559bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2588unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2595 "Unexpected opcode");
2597 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2599 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2602 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2676bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2677 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2686 "Expected legal fixed length vector!");
2689 unsigned MaxELen = Subtarget.
getELen();
2723 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2730 "Expected to convert into a scalable vector!");
2731 assert(V.getValueType().isFixedLengthVector() &&
2732 "Expected a fixed length vector operand!");
2742 "Expected to convert into a fixed length vector!");
2743 assert(V.getValueType().isScalableVector() &&
2744 "Expected a scalable vector operand!");
2772 const auto [MinVLMAX, MaxVLMAX] =
2774 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2780static std::pair<SDValue, SDValue>
2789static std::pair<SDValue, SDValue>
2802static std::pair<SDValue, SDValue>
2819std::pair<unsigned, unsigned>
2835 return std::make_pair(MinVLMAX, MaxVLMAX);
2847 EVT VT,
unsigned DefinedValues)
const {
2861 std::tie(LMul, Fractional) =
2864 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2866 Cost = (LMul * DLenFactor);
2911 MVT DstVT =
Op.getSimpleValueType();
2912 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2920 Src.getValueType() == MVT::bf16) {
2927 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2935 Opc,
DL, DstVT, Src,
2949 MVT SrcVT = Src.getSimpleValueType();
2955 if (SatVT != DstEltVT)
2959 if (SrcEltSize > (2 * DstEltSize))
2962 MVT DstContainerVT = DstVT;
2963 MVT SrcContainerVT = SrcVT;
2969 "Expected same element count");
2978 {Src, Src, DAG.getCondCode(ISD::SETNE),
2979 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2983 if (DstEltSize > (2 * SrcEltSize)) {
2997 Res, DAG.
getUNDEF(DstContainerVT), VL);
3009 case ISD::VP_FROUNDEVEN:
3013 case ISD::VP_FROUNDTOZERO:
3017 case ISD::VP_FFLOOR:
3025 case ISD::VP_FROUND:
3041 MVT VT =
Op.getSimpleValueType();
3048 MVT ContainerVT = VT;
3055 if (
Op->isVPOpcode()) {
3056 Mask =
Op.getOperand(1);
3060 VL =
Op.getOperand(2);
3082 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3096 switch (
Op.getOpcode()) {
3102 case ISD::VP_FFLOOR:
3105 case ISD::VP_FROUND:
3106 case ISD::VP_FROUNDEVEN:
3107 case ISD::VP_FROUNDTOZERO: {
3123 case ISD::VP_FNEARBYINT:
3136 Src, Src, Mask, VL);
3151 MVT VT =
Op.getSimpleValueType();
3155 MVT ContainerVT = VT;
3167 MVT MaskVT = Mask.getSimpleValueType();
3170 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3171 DAG.getUNDEF(MaskVT), Mask, VL});
3175 {Chain, Src, Src, Src, Unorder, VL});
3176 Chain = Src.getValue(1);
3192 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3204 switch (
Op.getOpcode()) {
3215 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3221 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3225 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3234 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3235 Truncated, Mask, VL);
3241 Src, Src, Mask, VL);
3251 MVT VT =
Op.getSimpleValueType();
3279 MVT VT =
Op.getSimpleValueType();
3284 MVT ContainerVT = VT;
3306 if (
Merge.isUndef())
3318 if (
Merge.isUndef())
3327 "Unexpected vector MVT");
3355 return std::nullopt;
3370 unsigned EltSizeInBits) {
3373 return std::nullopt;
3374 bool IsInteger =
Op.getValueType().isInteger();
3376 std::optional<unsigned> SeqStepDenom;
3377 std::optional<int64_t> SeqStepNum, SeqAddend;
3378 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3379 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3384 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3386 if (Elt.isUndef()) {
3387 Elts[
Idx] = std::nullopt;
3391 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3396 return std::nullopt;
3397 Elts[
Idx] = *ExactInteger;
3410 unsigned IdxDiff =
Idx - PrevElt->second;
3411 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3419 int64_t Remainder = ValDiff % IdxDiff;
3421 if (Remainder != ValDiff) {
3424 return std::nullopt;
3430 SeqStepNum = ValDiff;
3431 else if (ValDiff != SeqStepNum)
3432 return std::nullopt;
3435 SeqStepDenom = IdxDiff;
3436 else if (IdxDiff != *SeqStepDenom)
3437 return std::nullopt;
3441 if (!PrevElt || PrevElt->first != *Elt)
3442 PrevElt = std::make_pair(*Elt,
Idx);
3446 if (!SeqStepNum || !SeqStepDenom)
3447 return std::nullopt;
3455 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3456 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3459 else if (Addend != SeqAddend)
3460 return std::nullopt;
3463 assert(SeqAddend &&
"Must have an addend if we have a step");
3465 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3486 MVT ContainerVT = VT;
3514 MVT VT =
Op.getSimpleValueType();
3526 unsigned MostCommonCount = 0;
3528 unsigned NumUndefElts =
3536 unsigned NumScalarLoads = 0;
3542 ValueCounts.
insert(std::make_pair(V, 0));
3543 unsigned &Count = ValueCounts[V];
3545 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3546 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3551 if (++Count >= MostCommonCount) {
3553 MostCommonCount = Count;
3557 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3558 unsigned NumDefElts = NumElts - NumUndefElts;
3559 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3565 ((MostCommonCount > DominantValueCountThreshold) ||
3578 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3579 LastOp != DominantValue) {
3588 Processed.insert(LastOp);
3593 const SDValue &V = OpIdx.value();
3594 if (V.isUndef() || !Processed.insert(V).second)
3596 if (ValueCounts[V] == 1) {
3605 return DAG.getConstant(V == V1, DL, XLenVT);
3621 MVT VT =
Op.getSimpleValueType();
3651 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3652 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3660 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3661 MVT IntegerViaVecVT =
3666 unsigned BitPos = 0, IntegerEltIdx = 0;
3669 for (
unsigned I = 0;
I < NumElts;) {
3671 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3672 Bits |= ((
uint64_t)BitValue << BitPos);
3678 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3679 if (NumViaIntegerBits <= 32)
3680 Bits = SignExtend64<32>(Bits);
3682 Elts[IntegerEltIdx] = Elt;
3691 if (NumElts < NumViaIntegerBits) {
3695 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3723 int64_t StepNumerator = SimpleVID->StepNumerator;
3724 unsigned StepDenominator = SimpleVID->StepDenominator;
3725 int64_t Addend = SimpleVID->Addend;
3727 assert(StepNumerator != 0 &&
"Invalid step");
3728 bool Negate =
false;
3729 int64_t SplatStepVal = StepNumerator;
3733 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3735 Negate = StepNumerator < 0;
3737 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3744 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3745 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3747 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3750 MVT VIDContainerVT =
3758 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3759 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3761 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3763 if (StepDenominator != 1) {
3768 if (Addend != 0 || Negate) {
3787 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3788 "Unexpected sequence type");
3792 unsigned ViaVecLen =
3796 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3799 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3800 const auto &SeqV = OpIdx.value();
3801 if (!SeqV.isUndef())
3803 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3808 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3809 SplatValue = SignExtend64<32>(SplatValue);
3831 const auto *BV = cast<BuildVectorSDNode>(
Op);
3834 BV->getRepeatedSequence(Sequence) &&
3835 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3836 unsigned SeqLen = Sequence.size();
3838 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3839 ViaIntVT == MVT::i64) &&
3840 "Unexpected sequence type");
3845 const unsigned RequiredVL = NumElts / SeqLen;
3846 const unsigned ViaVecLen =
3848 NumElts : RequiredVL;
3851 unsigned EltIdx = 0;
3852 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3856 for (
const auto &SeqV : Sequence) {
3857 if (!SeqV.isUndef())
3859 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3865 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3866 SplatValue = SignExtend64<32>(SplatValue);
3872 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3873 "Unexpected bitcast sequence");
3874 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3877 MVT ViaContainerVT =
3884 if (ViaVecLen != RequiredVL)
3901 if (EltBitSize - SignBits < 8) {
3905 Source, DAG, Subtarget);
3926 return RISCV::PACKH;
3928 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
3943 MVT VT =
Op.getSimpleValueType();
3951 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
3956 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
3970 if (Subtarget.hasStdExtZbkb())
3975 ElemDL, XLenVT,
A,
B),
3982 Flags.setDisjoint(
true);
3988 NewOperands.
reserve(NumElts / 2);
3990 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
4000 MVT VT =
Op.getSimpleValueType();
4011 MVT VT =
Op.getSimpleValueType();
4016 !Subtarget.hasStdExtZfhmin())
4088 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
4092 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4115 unsigned NumUndefElts =
4117 unsigned NumDefElts = NumElts - NumUndefElts;
4118 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4125 for (
unsigned i = 0; i < NumElts; i++) {
4127 if (i < NumElts / 2) {
4134 bool SelectMaskVal = (i < NumElts / 2);
4137 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4138 MaskVals.
size() == NumElts);
4173 unsigned UndefCount = 0;
4180 LinearBudget -= PerSlideCost;
4183 LinearBudget -= PerSlideCost;
4186 LinearBudget -= PerSlideCost;
4189 if (LinearBudget < 0)
4194 "Illegal type which will result in reserved encoding");
4219 Vec,
Offset, Mask, VL, Policy);
4232 Vec,
Offset, Mask, VL, Policy);
4242 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4243 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4244 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4247 if ((LoC >> 31) == HiC)
4258 (isa<RegisterSDNode>(VL) &&
4259 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4261 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4276 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4277 Hi.getConstantOperandVal(1) == 31)
4296 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4308 bool HasPassthru = Passthru && !Passthru.
isUndef();
4309 if (!HasPassthru && !Passthru)
4317 if (Scalar.getValueType().bitsLE(XLenVT)) {
4324 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4328 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4329 "Unexpected scalar for splat lowering!");
4353 SDValue ExtractedVal = Scalar.getOperand(0);
4358 MVT ExtractedContainerVT = ExtractedVT;
4361 DAG, ExtractedContainerVT, Subtarget);
4363 ExtractedVal, DAG, Subtarget);
4365 if (ExtractedContainerVT.
bitsLE(VT))
4380 if (!Scalar.getValueType().bitsLE(XLenVT))
4383 VT,
DL, DAG, Subtarget);
4391 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4417 if (Src != V2.getOperand(0))
4421 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4426 V2.getConstantOperandVal(1) != Mask.size())
4430 if (Mask[0] != 0 && Mask[0] != 1)
4435 for (
unsigned i = 1; i != Mask.size(); ++i)
4436 if (Mask[i] != Mask[i - 1] + 2)
4454 int Size = Mask.size();
4456 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4462 EvenSrc = StartIndexes[0];
4463 OddSrc = StartIndexes[1];
4466 if (EvenSrc != 0 && OddSrc != 0)
4476 int HalfNumElts = NumElts / 2;
4477 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4493 int Size = Mask.size();
4505 for (
int i = 0; i !=
Size; ++i) {
4511 int StartIdx = i - (M %
Size);
4519 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4522 Rotation = CandidateRotation;
4523 else if (Rotation != CandidateRotation)
4528 int MaskSrc = M <
Size ? 0 : 1;
4533 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4538 TargetSrc = MaskSrc;
4539 else if (TargetSrc != MaskSrc)
4546 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4547 assert((LoSrc >= 0 || HiSrc >= 0) &&
4548 "Failed to find a rotated input vector!");
4563 MVT ContainerVT = VT;
4566 assert(Src.getSimpleValueType().isFixedLengthVector());
4570 MVT SrcContainerVT =
4583 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4590 unsigned Shift = EvenElts ? 0 : EltBits;
4596 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4622 auto findNonEXTRACT_SUBVECTORParent =
4623 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4628 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4629 Offset += Parent.getConstantOperandVal(1);
4630 Parent = Parent.getOperand(0);
4632 return std::make_pair(Parent,
Offset);
4635 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4636 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4645 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4646 if (NewMask[i] == -1)
4649 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4650 NewMask[i] = NewMask[i] + V1IndexOffset;
4654 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4660 if (NewMask[0] <= 0)
4664 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4665 if (NewMask[i - 1] + 1 != NewMask[i])
4669 MVT SrcVT = Src.getSimpleValueType();
4700 int NumSubElts,
Index;
4705 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4706 SDValue InPlace = OpsSwapped ? V2 : V1;
4707 SDValue ToInsert = OpsSwapped ? V1 : V2;
4717 if (NumSubElts +
Index >= (
int)NumElts)
4731 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4743 bool OpsSwapped =
false;
4744 if (!isa<BuildVectorSDNode>(V1)) {
4745 if (!isa<BuildVectorSDNode>(V2))
4750 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4758 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4759 for (
unsigned i = S; i != E; ++i)
4760 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4766 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4767 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4770 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4772 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4777 auto OpCode = IsVSlidedown ?
4782 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4785 Splat, TrueMask, VL);
4796 MVT VecContainerVT = VecVT;
4813 MVT WideContainerVT = WideVT;
4819 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4832 }
else if (Subtarget.hasStdExtZvbb()) {
4837 OffsetVec, Passthru, Mask, VL);
4840 Interleaved, EvenV, Passthru, Mask, VL);
4848 Interleaved, OffsetVec, Passthru, Mask, VL);
4856 OddV, Passthru, Mask, VL);
4862 OddV, AllOnesVec, Passthru, Mask, VL);
4870 Interleaved, OddsMul, Passthru, Mask, VL);
4877 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4923 if (ViaEltSize > NumElts)
4932 if (ViaEltSize > NumElts)
4938 if (ViaEltSize > NumElts)
4947 MVT &RotateVT,
unsigned &RotateAmt) {
4953 unsigned NumSubElts;
4955 NumElts, NumSubElts, RotateAmt))
4958 NumElts / NumSubElts);
5020 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
5023 OutMasks(VRegsPerSrc, {-1, {}});
5028 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
5029 int DstVecIdx = DstIdx / ElemsPerVReg;
5030 int DstSubIdx = DstIdx % ElemsPerVReg;
5031 int SrcIdx = Mask[DstIdx];
5032 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
5034 int SrcVecIdx = SrcIdx / ElemsPerVReg;
5035 int SrcSubIdx = SrcIdx % ElemsPerVReg;
5036 if (OutMasks[DstVecIdx].first == -1)
5037 OutMasks[DstVecIdx].first = SrcVecIdx;
5038 if (OutMasks[DstVecIdx].first != SrcVecIdx)
5044 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
5045 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
5059 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
5060 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
5061 if (SrcVecIdx == -1)
5063 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
5070 unsigned InsertIdx = DstVecIdx * NumOpElts;
5083 MVT VT =
Op.getSimpleValueType();
5098 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5122 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5123 V = V.getOperand(
Offset / OpElements);
5129 auto *Ld = cast<LoadSDNode>(V);
5139 SDValue Ops[] = {Ld->getChain(),
5153 MVT SplatVT = ContainerVT;
5156 if (SVT == MVT::f16 && !Subtarget.hasStdExtZfh()) {
5165 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5166 Ld->getPointerInfo().getWithOffset(
Offset),
5167 Ld->getOriginalAlign(),
5171 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5172 Ld->getOriginalAlign(),
5173 Ld->getMemOperand()->getFlags());
5185 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5188 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5210 if (Subtarget.hasStdExtZvkb())
5221 LoV = LoSrc == 0 ? V1 : V2;
5225 HiV = HiSrc == 0 ? V1 : V2;
5231 unsigned InvRotate = NumElts - Rotation;
5241 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5261 int EvenSrc, OddSrc;
5266 int Size = Mask.size();
5268 assert(EvenSrc >= 0 &&
"Undef source?");
5269 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5273 assert(OddSrc >= 0 &&
"Undef source?");
5274 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5283 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5292 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5321 MVT IndexContainerVT =
5326 for (
int MaskIndex : Mask) {
5327 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5336 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5346 for (
int MaskIndex : Mask) {
5347 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5348 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5350 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5365 for (
int MaskIndex : Mask) {
5366 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5370 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5402RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5404 MVT VT =
Op.getSimpleValueType();
5408 MVT ContainerVT = VT;
5411 if (
Op->isVPOpcode()) {
5412 Mask =
Op.getOperand(1);
5416 VL =
Op.getOperand(2);
5422 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5424 FloatEltVT = MVT::f32;
5431 "Expected legal float type!");
5438 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5441 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5446 if (FloatVT.
bitsGT(VT)) {
5447 if (
Op->isVPOpcode())
5448 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5457 if (!
Op->isVPOpcode())
5461 MVT ContainerFloatVT =
5464 Src, Mask, RTZRM, VL);
5471 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5475 if (
Op->isVPOpcode()) {
5484 else if (IntVT.
bitsGT(VT))
5489 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5494 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5495 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5500 unsigned Adjust = ExponentBias + (EltSize - 1);
5502 if (
Op->isVPOpcode())
5512 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5513 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5523 MVT SrcVT =
Source.getSimpleValueType();
5532 SrcVT = ContainerVT;
5545 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5562 auto *
Load = cast<LoadSDNode>(
Op);
5563 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5566 Load->getMemoryVT(),
5567 *
Load->getMemOperand()))
5571 MVT VT =
Op.getSimpleValueType();
5573 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5574 "Unexpected unaligned RVV load type");
5578 "Expecting equally-sized RVV vector types to be legal");
5580 Load->getPointerInfo(),
Load->getOriginalAlign(),
5581 Load->getMemOperand()->getFlags());
5591 auto *
Store = cast<StoreSDNode>(
Op);
5592 assert(Store &&
Store->getValue().getValueType().isVector() &&
5593 "Expected vector store");
5596 Store->getMemoryVT(),
5597 *
Store->getMemOperand()))
5604 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5605 "Unexpected unaligned RVV store type");
5609 "Expecting equally-sized RVV vector types to be legal");
5610 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5612 Store->getPointerInfo(),
Store->getOriginalAlign(),
5613 Store->getMemOperand()->getFlags());
5618 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5620 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5647 unsigned ShiftAmt, AddOpc;
5664 if (Subtarget.hasStdExtZtso()) {
5687 "Unexpected custom legalisation");
5708 "Unexpected custom legalisation");
5723 "Unexpected custom legalisation");
5724 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5744 "Unexpected custom legalisation");
5760 MVT VT =
Op.getSimpleValueType();
5762 unsigned Check =
Op.getConstantOperandVal(1);
5763 unsigned TDCMask = 0;
5791 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5796 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5798 VL =
Op.getOperand(3);
5801 VL,
Op->getFlags());
5816 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5818 MVT MaskContainerVT =
5821 VL =
Op.getOperand(3);
5826 Mask, VL,
Op->getFlags());
5829 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5834 DAG.
getUNDEF(ContainerVT), Mask, VL});
5838 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5842 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5846 DAG.
getUNDEF(ContainerVT), Mask, VL});
5862 MVT VT =
Op.getSimpleValueType();
5889 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5896 MVT ContainerVT = VT;
5904 if (
Op->isVPOpcode()) {
5905 Mask =
Op.getOperand(2);
5909 VL =
Op.getOperand(3);
5917 {X, X, DAG.getCondCode(ISD::SETOEQ),
5918 DAG.getUNDEF(ContainerVT), Mask, VL});
5926 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5927 DAG.getUNDEF(ContainerVT), Mask, VL});
5937 DAG.
getUNDEF(ContainerVT), Mask, VL);
5945#define OP_CASE(NODE) \
5947 return RISCVISD::NODE##_VL;
5948#define VP_CASE(NODE) \
5949 case ISD::VP_##NODE: \
5950 return RISCVISD::NODE##_VL;
5952 switch (
Op.getOpcode()) {
6030 case ISD::VP_CTLZ_ZERO_UNDEF:
6033 case ISD::VP_CTTZ_ZERO_UNDEF:
6042 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6047 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6052 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6055 case ISD::VP_SELECT:
6064 case ISD::VP_SIGN_EXTEND:
6066 case ISD::VP_ZERO_EXTEND:
6068 case ISD::VP_FP_TO_SINT:
6070 case ISD::VP_FP_TO_UINT:
6073 case ISD::VP_FMINNUM:
6076 case ISD::VP_FMAXNUM:
6081 case ISD::VP_LLRINT:
6093 "not a RISC-V target specific op");
6099 "adding target specific op should update this function");
6119 "not a RISC-V target specific op");
6125 "adding target specific op should update this function");
6144 if (!
Op.getOperand(j).getValueType().isVector()) {
6145 LoOperands[j] =
Op.getOperand(j);
6146 HiOperands[j] =
Op.getOperand(j);
6149 std::tie(LoOperands[j], HiOperands[j]) =
6154 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6156 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6171 std::tie(LoOperands[j], HiOperands[j]) =
6175 if (!
Op.getOperand(j).getValueType().isVector()) {
6176 LoOperands[j] =
Op.getOperand(j);
6177 HiOperands[j] =
Op.getOperand(j);
6180 std::tie(LoOperands[j], HiOperands[j]) =
6185 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6187 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6197 auto [EVLLo, EVLHi] =
6198 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6202 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6204 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6222 if (!
Op.getOperand(j).getValueType().isVector()) {
6223 LoOperands[j] =
Op.getOperand(j);
6224 HiOperands[j] =
Op.getOperand(j);
6227 std::tie(LoOperands[j], HiOperands[j]) =
6232 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6235 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6244 switch (
Op.getOpcode()) {
6250 return lowerGlobalAddress(
Op, DAG);
6252 return lowerBlockAddress(
Op, DAG);
6254 return lowerConstantPool(
Op, DAG);
6256 return lowerJumpTable(
Op, DAG);
6258 return lowerGlobalTLSAddress(
Op, DAG);
6262 return lowerSELECT(
Op, DAG);
6264 return lowerBRCOND(
Op, DAG);
6266 return lowerVASTART(
Op, DAG);
6268 return lowerFRAMEADDR(
Op, DAG);
6270 return lowerRETURNADDR(
Op, DAG);
6277 return lowerShiftLeftParts(
Op, DAG);
6279 return lowerShiftRightParts(
Op, DAG,
true);
6281 return lowerShiftRightParts(
Op, DAG,
false);
6284 if (
Op.getValueType().isFixedLengthVector()) {
6285 assert(Subtarget.hasStdExtZvkb());
6286 return lowerToScalableOp(
Op, DAG);
6288 assert(Subtarget.hasVendorXTHeadBb() &&
6289 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6290 "Unexpected custom legalization");
6292 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6297 EVT VT =
Op.getValueType();
6301 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6307 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6308 Subtarget.hasStdExtZfbfmin()) {
6313 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6320 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6337 "Unexpected types");
6371 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6373 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6375 return LowerINTRINSIC_VOID(
Op, DAG);
6377 return LowerIS_FPCLASS(
Op, DAG);
6379 MVT VT =
Op.getSimpleValueType();
6381 assert(Subtarget.hasStdExtZvbb());
6382 return lowerToScalableOp(
Op, DAG);
6385 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6393 if (!
Op.getSimpleValueType().isVector())
6395 return lowerVectorTruncLike(
Op, DAG);
6398 if (
Op.getOperand(0).getValueType().isVector() &&
6399 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6400 return lowerVectorMaskExt(
Op, DAG, 1);
6403 if (
Op.getOperand(0).getValueType().isVector() &&
6404 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6405 return lowerVectorMaskExt(
Op, DAG, -1);
6408 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6410 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6412 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6414 MVT VT =
Op.getSimpleValueType();
6422 MVT ContainerVT = VT;
6428 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6435 MVT VT =
Op.getSimpleValueType();
6455 }
else if ((Val % 8) == 0) {
6471 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6472 Op.getOperand(1).getValueType() == MVT::i32) {
6484 if (
Op.getValueType() == MVT::nxv32f16 &&
6491 EVT VT =
Op.getValueType();
6494 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6496 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6502 if (!
Op.getValueType().isVector())
6504 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6508 EVT VT =
Op.getValueType();
6511 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6513 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6521 if (!
Op.getValueType().isVector())
6523 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6527 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6530 if (
Op.getValueType().isVector() &&
6531 Op.getValueType().getScalarType() == MVT::f16 &&
6534 if (
Op.getValueType() == MVT::nxv32f16)
6549 Op1.getValueType().isVector() &&
6550 Op1.getValueType().getScalarType() == MVT::f16 &&
6553 if (Op1.getValueType() == MVT::nxv32f16)
6558 Op1.getValueType().getVectorElementCount());
6561 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6571 MVT VT =
Op.getSimpleValueType();
6575 bool IsStrict =
Op->isStrictFPOpcode();
6576 SDValue Src =
Op.getOperand(0 + IsStrict);
6578 MVT SrcVT = Src.getSimpleValueType();
6583 "Unexpected vector element types");
6587 if (EltSize > (2 * SrcEltSize)) {
6599 Op.getOperand(0), Ext);
6603 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6608 auto [FExt, Chain] =
6610 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6617 if (SrcEltSize > (2 * EltSize)) {
6620 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6625 Op.getOperand(0), Src);
6640 Op.getOperand(0), Src);
6654 unsigned RVVOpc = 0;
6655 switch (
Op.getOpcode()) {
6687 "Expected same element count");
6694 Op.getOperand(0), Src, Mask, VL);
6698 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6713 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6720 MVT VT =
Op.getSimpleValueType();
6742 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6758 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6778 return lowerVECREDUCE(
Op, DAG);
6782 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6783 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6784 return lowerVECREDUCE(
Op, DAG);
6791 return lowerFPVECREDUCE(
Op, DAG);
6792 case ISD::VP_REDUCE_ADD:
6793 case ISD::VP_REDUCE_UMAX:
6794 case ISD::VP_REDUCE_SMAX:
6795 case ISD::VP_REDUCE_UMIN:
6796 case ISD::VP_REDUCE_SMIN:
6797 case ISD::VP_REDUCE_FADD:
6798 case ISD::VP_REDUCE_SEQ_FADD:
6799 case ISD::VP_REDUCE_FMIN:
6800 case ISD::VP_REDUCE_FMAX:
6801 case ISD::VP_REDUCE_FMINIMUM:
6802 case ISD::VP_REDUCE_FMAXIMUM:
6803 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6807 return lowerVPREDUCE(
Op, DAG);
6808 case ISD::VP_REDUCE_AND:
6809 case ISD::VP_REDUCE_OR:
6810 case ISD::VP_REDUCE_XOR:
6811 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6812 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6813 return lowerVPREDUCE(
Op, DAG);
6814 case ISD::VP_CTTZ_ELTS:
6815 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6816 return lowerVPCttzElements(
Op, DAG);
6820 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6823 return lowerINSERT_SUBVECTOR(
Op, DAG);
6825 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6827 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6829 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6831 return lowerSTEP_VECTOR(
Op, DAG);
6833 return lowerVECTOR_REVERSE(
Op, DAG);
6835 return lowerVECTOR_SPLICE(
Op, DAG);
6839 if ((
Op.getValueType().getScalarType() == MVT::f16 &&
6843 (
Op.getValueType().getScalarType() == MVT::bf16 &&
6845 if (
Op.getValueType() == MVT::nxv32f16 ||
6846 Op.getValueType() == MVT::nxv32bf16)
6858 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6859 return lowerVectorMaskSplat(
Op, DAG);
6867 MVT VT =
Op.getSimpleValueType();
6868 MVT ContainerVT = VT;
6886 Op->ops().take_front(HalfNumOps));
6888 Op->ops().drop_front(HalfNumOps));
6892 unsigned NumOpElts =
6893 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6896 SDValue SubVec = OpIdx.value();
6907 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6909 if (
Op.getValueType().isFixedLengthVector())
6910 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6913 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6915 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6916 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6920 return lowerMaskedLoad(
Op, DAG);
6923 return lowerMaskedStore(
Op, DAG);
6932 EVT VT =
Op.getValueType();
6943 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6945 MVT VT =
Op.getSimpleValueType();
6950 "Unexpected CondCode");
6958 if (isa<ConstantSDNode>(
RHS)) {
6959 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6960 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6979 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6984 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
7000 return lowerToScalableOp(
Op, DAG);
7004 if (
Op.getSimpleValueType().isFixedLengthVector())
7005 return lowerToScalableOp(
Op, DAG);
7007 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
7008 "Unexpected custom legalisation");
7020 if (
Op.getValueType() == MVT::nxv32f16 &&
7033 return lowerToScalableOp(
Op, DAG);
7036 if (!
Op.getValueType().isVector())
7038 return lowerToScalableOp(
Op, DAG);
7041 if (!
Op.getValueType().isVector())
7043 return lowerToScalableOp(
Op, DAG);
7047 EVT VT =
Op->getValueType(0);
7062 return lowerABS(
Op, DAG);
7067 if (Subtarget.hasStdExtZvbb())
7068 return lowerToScalableOp(
Op, DAG);
7070 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7072 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
7074 if (
Op.getValueType() == MVT::nxv32f16 &&
7078 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
7085 if (
Op.getValueType() == MVT::nxv32f16 &&
7089 return lowerToScalableOp(
Op, DAG);
7092 return lowerVectorStrictFSetcc(
Op, DAG);
7102 case ISD::VP_GATHER:
7103 return lowerMaskedGather(
Op, DAG);
7105 case ISD::VP_SCATTER:
7106 return lowerMaskedScatter(
Op, DAG);
7108 return lowerGET_ROUNDING(
Op, DAG);
7110 return lowerSET_ROUNDING(
Op, DAG);
7112 return lowerEH_DWARF_CFA(
Op, DAG);
7113 case ISD::VP_SELECT:
7122 case ISD::VP_UADDSAT:
7123 case ISD::VP_USUBSAT:
7124 case ISD::VP_SADDSAT:
7125 case ISD::VP_SSUBSAT:
7127 case ISD::VP_LLRINT:
7128 return lowerVPOp(
Op, DAG);
7132 return lowerLogicVPOp(
Op, DAG);
7141 case ISD::VP_FMINNUM:
7142 case ISD::VP_FMAXNUM:
7143 case ISD::VP_FCOPYSIGN:
7144 if (
Op.getValueType() == MVT::nxv32f16 &&
7152 return lowerVPOp(
Op, DAG);
7153 case ISD::VP_IS_FPCLASS:
7154 return LowerIS_FPCLASS(
Op, DAG);
7155 case ISD::VP_SIGN_EXTEND:
7156 case ISD::VP_ZERO_EXTEND:
7157 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7158 return lowerVPExtMaskOp(
Op, DAG);
7159 return lowerVPOp(
Op, DAG);
7160 case ISD::VP_TRUNCATE:
7161 return lowerVectorTruncLike(
Op, DAG);
7162 case ISD::VP_FP_EXTEND:
7163 case ISD::VP_FP_ROUND:
7164 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7165 case ISD::VP_SINT_TO_FP:
7166 case ISD::VP_UINT_TO_FP:
7167 if (
Op.getValueType().isVector() &&
7168 Op.getValueType().getScalarType() == MVT::f16 &&
7171 if (
Op.getValueType() == MVT::nxv32f16)
7183 case ISD::VP_FP_TO_SINT:
7184 case ISD::VP_FP_TO_UINT:
7186 Op1.getValueType().isVector() &&
7187 Op1.getValueType().getScalarType() == MVT::f16 &&
7190 if (Op1.getValueType() == MVT::nxv32f16)
7195 Op1.getValueType().getVectorElementCount());
7199 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7201 return lowerVPFPIntConvOp(
Op, DAG);
7203 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
7207 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7208 return lowerVPSetCCMaskOp(
Op, DAG);
7214 case ISD::VP_BITREVERSE:
7216 return lowerVPOp(
Op, DAG);
7218 case ISD::VP_CTLZ_ZERO_UNDEF:
7219 if (Subtarget.hasStdExtZvbb())
7220 return lowerVPOp(
Op, DAG);
7221 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7223 case ISD::VP_CTTZ_ZERO_UNDEF:
7224 if (Subtarget.hasStdExtZvbb())
7225 return lowerVPOp(
Op, DAG);
7226 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7228 return lowerVPOp(
Op, DAG);
7229 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7230 return lowerVPStridedLoad(
Op, DAG);
7231 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7232 return lowerVPStridedStore(
Op, DAG);
7234 case ISD::VP_FFLOOR:
7236 case ISD::VP_FNEARBYINT:
7237 case ISD::VP_FROUND:
7238 case ISD::VP_FROUNDEVEN:
7239 case ISD::VP_FROUNDTOZERO:
7240 if (
Op.getValueType() == MVT::nxv32f16 &&
7245 case ISD::VP_FMAXIMUM:
7246 case ISD::VP_FMINIMUM:
7247 if (
Op.getValueType() == MVT::nxv32f16 &&
7252 case ISD::EXPERIMENTAL_VP_SPLICE:
7253 return lowerVPSpliceExperimental(
Op, DAG);
7254 case ISD::EXPERIMENTAL_VP_REVERSE:
7255 return lowerVPReverseExperimental(
Op, DAG);
7256 case ISD::EXPERIMENTAL_VP_SPLAT:
7257 return lowerVPSplatExperimental(
Op, DAG);
7260 "llvm.clear_cache only needs custom lower on Linux targets");
7263 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7264 Op.getOperand(2), Flags,
DL);
7272 MakeLibCallOptions CallOptions;
7273 std::pair<SDValue, SDValue> CallResult =
7274 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7275 {Start,
End, Flags}, CallOptions,
DL, InChain);
7278 return CallResult.second;
7295 N->getOffset(), Flags);
7303template <
class NodeTy>
7305 bool IsLocal,
bool IsExternWeak)
const {
7315 if (IsLocal && !Subtarget.allowTaggedGlobals())
7377 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7386 return getAddr(
N, DAG);
7393 return getAddr(
N, DAG);
7400 return getAddr(
N, DAG);
7405 bool UseGOT)
const {
7469 Args.push_back(Entry);
7502 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7516 Addr = getStaticTLSAddr(
N, DAG,
false);
7519 Addr = getStaticTLSAddr(
N, DAG,
true);
7524 : getDynamicTLSAddr(
N, DAG);
7541 if (
LHS == LHS2 &&
RHS == RHS2) {
7546 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7554 return std::nullopt;
7562 MVT VT =
N->getSimpleValueType(0);
7592 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7595 if (~TrueVal == FalseVal) {
7635 if (Subtarget.hasShortForwardBranchOpt())
7638 unsigned SelOpNo = 0;
7648 unsigned ConstSelOpNo = 1;
7649 unsigned OtherSelOpNo = 2;
7650 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7655 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7656 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7660 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7661 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7667 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7669 std::swap(NewConstOps[0], NewConstOps[1]);
7681 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7683 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7686 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7687 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7696 MVT VT =
Op.getSimpleValueType();
7710 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7738 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7742 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7744 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7745 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7747 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7752 DL, VT, LHSVal, CondV);
7768 if (
Op.hasOneUse()) {
7769 unsigned UseOpc =
Op->use_begin()->getOpcode();
7778 return lowerSELECT(NewSel, DAG);
7807 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7828 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7832 if (TrueVal - 1 == FalseVal)
7834 if (TrueVal + 1 == FalseVal)
7841 RHS == TrueV && LHS == FalseV) {
7858 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7884 LHS, RHS, TargetCC,
Op.getOperand(2));
7902 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7914 int XLenInBytes = Subtarget.
getXLen() / 8;
7916 EVT VT =
Op.getValueType();
7919 unsigned Depth =
Op.getConstantOperandVal(0);
7921 int Offset = -(XLenInBytes * 2);
7937 int XLenInBytes = Subtarget.
getXLen() / 8;
7942 EVT VT =
Op.getValueType();
7944 unsigned Depth =
Op.getConstantOperandVal(0);
7946 int Off = -XLenInBytes;
7947 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7966 EVT VT =
Lo.getValueType();
8005 EVT VT =
Lo.getValueType();
8056 MVT VT =
Op.getSimpleValueType();
8082 MVT VecVT =
Op.getSimpleValueType();
8084 "Unexpected SPLAT_VECTOR_PARTS lowering");
8090 MVT ContainerVT = VecVT;
8110 int64_t ExtTrueVal)
const {
8112 MVT VecVT =
Op.getSimpleValueType();
8115 assert(Src.getValueType().isVector() &&
8116 Src.getValueType().getVectorElementType() == MVT::i1);
8137 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8139 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
8142 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
8147SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
8149 MVT ExtVT =
Op.getSimpleValueType();
8153 MVT VT =
Op.getOperand(0).getSimpleValueType();
8179 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8181 EVT MaskVT =
Op.getValueType();
8184 "Unexpected type for vector mask lowering");
8186 MVT VecVT = Src.getSimpleValueType();
8190 VL =
Op.getOperand(2);
8193 MVT ContainerVT = VecVT;
8199 MVT MaskContainerVT =
8206 std::tie(Mask, VL) =
8214 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8216 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8220 DAG.
getUNDEF(ContainerVT), Mask, VL);
8223 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8231 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8234 MVT VT =
Op.getSimpleValueType();
8236 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8240 return lowerVectorMaskTruncLike(
Op, DAG);
8248 MVT SrcVT = Src.getSimpleValueType();
8253 "Unexpected vector truncate lowering");
8255 MVT ContainerVT = SrcVT;
8259 VL =
Op.getOperand(2);
8272 std::tie(Mask, VL) =
8283 }
while (SrcEltVT != DstEltVT);
8292RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8297 MVT VT =
Op.getSimpleValueType();
8298 MVT SrcVT = Src.getSimpleValueType();
8299 MVT ContainerVT = VT;
8322 Chain, Src, Mask, VL);
8323 Chain = Src.getValue(1);
8330 Chain, Src, Mask, VL);
8341RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8344 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8351 MVT VT =
Op.getSimpleValueType();
8353 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8356 MVT SrcVT = Src.getSimpleValueType();
8358 bool IsDirectExtend =
8366 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8369 MVT ContainerVT = VT;
8373 VL =
Op.getOperand(2);
8387 std::tie(Mask, VL) =
8393 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8399 unsigned InterConvOpc =
8404 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8406 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8417static std::optional<MVT>
8423 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8425 if (MaxIdx < MinVLMAX)
8427 else if (MaxIdx < MinVLMAX * 2)
8429 else if (MaxIdx < MinVLMAX * 4)
8434 return std::nullopt;
8447 MVT VecVT =
Op.getSimpleValueType();
8461 MVT ContainerVT = VecVT;
8470 MVT OrigContainerVT = ContainerVT;
8473 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8474 const unsigned OrigIdx = IdxC->getZExtValue();
8477 DL, DAG, Subtarget)) {
8478 ContainerVT = *ShrunkVT;
8487 VLEN && ContainerVT.
bitsGT(M1VT)) {
8490 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8491 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8492 unsigned ExtractIdx =
8511 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8512 const auto *CVal = cast<ConstantSDNode>(Val);
8513 if (isInt<32>(CVal->getSExtValue())) {
8514 IsLegalInsert =
true;
8523 if (IsLegalInsert) {
8529 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8545 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8546 MVT I32ContainerVT =
8557 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8562 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8564 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8569 ValInVec, AlignedIdx);
8579 DAG.
getUNDEF(I32ContainerVT), ValLo,
8580 I32Mask, InsertI64VL);
8582 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8583 I32Mask, InsertI64VL);
8585 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8598 Idx, Mask, InsertVL, Policy);
8602 Slideup, AlignedIdx);
8617 EVT EltVT =
Op.getValueType();
8624 MVT ContainerVT = VecVT;
8640 unsigned WidenVecLen;
8643 unsigned MaxEEW = Subtarget.
getELen();
8648 "the number of elements should be power of 2");
8652 ExtractBitIdx =
Idx;
8654 WideEltVT = LargestEltVT;
8657 ExtractElementIdx = DAG.
getNode(
8668 Vec, ExtractElementIdx);
8684 MVT ContainerVT = VecVT;
8695 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8696 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8698 unsigned OrigIdx = IdxC->getZExtValue();
8701 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8702 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8703 unsigned ExtractIdx =
8713 std::optional<uint64_t> MaxIdx;
8716 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8717 MaxIdx = IdxC->getZExtValue();
8719 if (
auto SmallerVT =
8721 ContainerVT = *SmallerVT;
8768 "Unexpected opcode");
8775 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8780 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8781 if (!
II || !
II->hasScalarOperand())
8784 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
8797 if (OpVT.
bitsLT(XLenVT)) {
8804 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8814 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
8815 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8818 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8829 case Intrinsic::riscv_vslide1up:
8830 case Intrinsic::riscv_vslide1down:
8831 case Intrinsic::riscv_vslide1up_mask:
8832 case Intrinsic::riscv_vslide1down_mask: {
8835 bool IsMasked = NumOps == 7;
8841 std::tie(ScalarLo, ScalarHi) =
8849 if (isa<ConstantSDNode>(AVL)) {
8850 const auto [MinVLMAX, MaxVLMAX] =
8854 if (AVLInt <= MinVLMAX) {
8856 }
else if (AVLInt >= 2 * MaxVLMAX) {
8863 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8897 if (IntNo == Intrinsic::riscv_vslide1up ||
8898 IntNo == Intrinsic::riscv_vslide1up_mask) {
8900 ScalarHi, I32Mask, I32VL);
8902 ScalarLo, I32Mask, I32VL);
8905 ScalarLo, I32Mask, I32VL);
8907 ScalarHi, I32Mask, I32VL);
8956 const unsigned ElementWidth = 8;
8961 [[maybe_unused]]
unsigned MinVF =
8964 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8968 bool Fractional = VF < LMul1VF;
8969 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8990 MVT ContainerVT = OpVT;
9017 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9021 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9022 if (!
II || !
II->hasScalarOperand())
9025 unsigned SplatOp =
II->ScalarOperand + 1;
9038 if (OpVT.
bitsLT(XLenVT)) {
9041 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9054 EVT ValType = V.getValueType();
9055 if (ValType.isVector() && ValType.isFloatingPoint()) {
9058 ValType.getVectorElementCount());
9061 if (ValType.isFixedLengthVector()) {
9063 DAG, V.getSimpleValueType(), Subtarget);
9079 unsigned IntNo =
Op.getConstantOperandVal(0);
9086 case Intrinsic::thread_pointer: {
9090 case Intrinsic::riscv_orc_b:
9091 case Intrinsic::riscv_brev8:
9092 case Intrinsic::riscv_sha256sig0:
9093 case Intrinsic::riscv_sha256sig1:
9094 case Intrinsic::riscv_sha256sum0:
9095 case Intrinsic::riscv_sha256sum1:
9096 case Intrinsic::riscv_sm3p0:
9097 case Intrinsic::riscv_sm3p1: {
9117 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9119 case Intrinsic::riscv_sm4ks:
9120 case Intrinsic::riscv_sm4ed: {
9130 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
9134 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
9137 case Intrinsic::riscv_zip:
9138 case Intrinsic::riscv_unzip: {
9141 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9143 case Intrinsic::riscv_mopr: {
9156 case Intrinsic::riscv_moprr: {
9168 Op.getOperand(2),
Op.getOperand(3));
9170 case Intrinsic::riscv_clmul:
9181 case Intrinsic::riscv_clmulh:
9182 case Intrinsic::riscv_clmulr: {
9200 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9202 case Intrinsic::experimental_get_vector_length:
9204 case Intrinsic::experimental_cttz_elts:
9206 case Intrinsic::riscv_vmv_x_s: {
9210 case Intrinsic::riscv_vfmv_f_s:
9213 case Intrinsic::riscv_vmv_v_x:
9215 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9217 case Intrinsic::riscv_vfmv_v_f:
9219 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9220 case Intrinsic::riscv_vmv_s_x: {
9223 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9226 Op.getOperand(1), Scalar,
Op.getOperand(3));
9229 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9246 MVT VT =
Op.getSimpleValueType();
9251 if (
Op.getOperand(1).isUndef())
9267 case Intrinsic::riscv_vfmv_s_f:
9269 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9271 case Intrinsic::riscv_vaesdf_vv:
9272 case Intrinsic::riscv_vaesdf_vs:
9273 case Intrinsic::riscv_vaesdm_vv:
9274 case Intrinsic::riscv_vaesdm_vs:
9275 case Intrinsic::riscv_vaesef_vv:
9276 case Intrinsic::riscv_vaesef_vs:
9277 case Intrinsic::riscv_vaesem_vv:
9278 case Intrinsic::riscv_vaesem_vs:
9279 case Intrinsic::riscv_vaeskf1:
9280 case Intrinsic::riscv_vaeskf2:
9281 case Intrinsic::riscv_vaesz_vs:
9282 case Intrinsic::riscv_vsm4k:
9283 case Intrinsic::riscv_vsm4r_vv:
9284 case Intrinsic::riscv_vsm4r_vs: {
9285 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9286 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9287 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9292 case Intrinsic::riscv_vsm3c:
9293 case Intrinsic::riscv_vsm3me: {
9294 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9295 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9300 case Intrinsic::riscv_vsha2ch:
9301 case Intrinsic::riscv_vsha2cl:
9302 case Intrinsic::riscv_vsha2ms: {
9303 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9304 !Subtarget.hasStdExtZvknhb())
9306 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9307 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9308 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9312 case Intrinsic::riscv_sf_vc_v_x:
9313 case Intrinsic::riscv_sf_vc_v_i:
9314 case Intrinsic::riscv_sf_vc_v_xv:
9315 case Intrinsic::riscv_sf_vc_v_iv:
9316 case Intrinsic::riscv_sf_vc_v_vv:
9317 case Intrinsic::riscv_sf_vc_v_fv:
9318 case Intrinsic::riscv_sf_vc_v_xvv:
9319 case Intrinsic::riscv_sf_vc_v_ivv:
9320 case Intrinsic::riscv_sf_vc_v_vvv:
9321 case Intrinsic::riscv_sf_vc_v_fvv:
9322 case Intrinsic::riscv_sf_vc_v_xvw:
9323 case Intrinsic::riscv_sf_vc_v_ivw:
9324 case Intrinsic::riscv_sf_vc_v_vvw:
9325 case Intrinsic::riscv_sf_vc_v_fvw: {
9326 MVT VT =
Op.getSimpleValueType();
9363 MVT VT =
Op.getSimpleValueType();
9367 if (VT.isFloatingPoint()) {
9372 if (VT.isFixedLengthVector())
9382 if (VT.isFixedLengthVector())
9384 if (VT.isFloatingPoint())
9403 unsigned IntNo =
Op.getConstantOperandVal(1);
9407 case Intrinsic::riscv_masked_strided_load: {
9416 MVT VT =
Op->getSimpleValueType(0);
9417 MVT ContainerVT = VT;
9430 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9442 ScalarVT,
Load->getMemOperand());
9448 Load->getMemOperand());
9453 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9458 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9460 Ops.push_back(PassThru);
9462 Ops.push_back(Stride);
9464 Ops.push_back(Mask);
9469 Ops.push_back(Policy);
9475 Load->getMemoryVT(),
Load->getMemOperand());
9476 Chain =
Result.getValue(1);
9482 case Intrinsic::riscv_seg2_load:
9483 case Intrinsic::riscv_seg3_load:
9484 case Intrinsic::riscv_seg4_load:
9485 case Intrinsic::riscv_seg5_load:
9486 case Intrinsic::riscv_seg6_load:
9487 case Intrinsic::riscv_seg7_load:
9488 case Intrinsic::riscv_seg8_load: {
9491 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9492 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9493 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9494 Intrinsic::riscv_vlseg8};
9495 unsigned NF =
Op->getNumValues() - 1;
9496 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9498 MVT VT =
Op->getSimpleValueType(0);
9504 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9506 ContainerVTs.push_back(MVT::Other);
9514 Load->getMemoryVT(),
Load->getMemOperand());
9516 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9522 case Intrinsic::riscv_sf_vc_v_x_se:
9524 case Intrinsic::riscv_sf_vc_v_i_se:
9526 case Intrinsic::riscv_sf_vc_v_xv_se:
9528 case Intrinsic::riscv_sf_vc_v_iv_se:
9530 case Intrinsic::riscv_sf_vc_v_vv_se:
9532 case Intrinsic::riscv_sf_vc_v_fv_se:
9534 case Intrinsic::riscv_sf_vc_v_xvv_se:
9536 case Intrinsic::riscv_sf_vc_v_ivv_se:
9538 case Intrinsic::riscv_sf_vc_v_vvv_se:
9540 case Intrinsic::riscv_sf_vc_v_fvv_se:
9542 case Intrinsic::riscv_sf_vc_v_xvw_se:
9544 case Intrinsic::riscv_sf_vc_v_ivw_se:
9546 case Intrinsic::riscv_sf_vc_v_vvw_se:
9548 case Intrinsic::riscv_sf_vc_v_fvw_se:
9557 unsigned IntNo =
Op.getConstantOperandVal(1);
9561 case Intrinsic::riscv_masked_strided_store: {
9572 MVT ContainerVT = VT;
9586 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9589 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9599 Ops,
Store->getMemoryVT(),
9600 Store->getMemOperand());
9602 case Intrinsic::riscv_seg2_store:
9603 case Intrinsic::riscv_seg3_store:
9604 case Intrinsic::riscv_seg4_store:
9605 case Intrinsic::riscv_seg5_store:
9606 case Intrinsic::riscv_seg6_store:
9607 case Intrinsic::riscv_seg7_store:
9608 case Intrinsic::riscv_seg8_store: {
9611 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9612 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9613 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9614 Intrinsic::riscv_vsseg8};
9617 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9619 MVT VT =
Op->getOperand(2).getSimpleValueType();
9627 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9629 for (
unsigned i = 0; i < NF; i++)
9631 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9636 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9638 case Intrinsic::riscv_sf_vc_xv_se:
9640 case Intrinsic::riscv_sf_vc_iv_se:
9642 case Intrinsic::riscv_sf_vc_vv_se:
9644 case Intrinsic::riscv_sf_vc_fv_se:
9646 case Intrinsic::riscv_sf_vc_xvv_se:
9648 case Intrinsic::riscv_sf_vc_ivv_se:
9650 case Intrinsic::riscv_sf_vc_vvv_se:
9652 case Intrinsic::riscv_sf_vc_fvv_se:
9654 case Intrinsic::riscv_sf_vc_xvw_se:
9656 case Intrinsic::riscv_sf_vc_ivw_se:
9658 case Intrinsic::riscv_sf_vc_vvw_se:
9660 case Intrinsic::riscv_sf_vc_fvw_se:
9668 switch (ISDOpcode) {
9671 case ISD::VP_REDUCE_ADD:
9674 case ISD::VP_REDUCE_UMAX:
9677 case ISD::VP_REDUCE_SMAX:
9680 case ISD::VP_REDUCE_UMIN:
9683 case ISD::VP_REDUCE_SMIN:
9686 case ISD::VP_REDUCE_AND:
9689 case ISD::VP_REDUCE_OR:
9692 case ISD::VP_REDUCE_XOR:
9695 case ISD::VP_REDUCE_FADD:
9697 case ISD::VP_REDUCE_SEQ_FADD:
9699 case ISD::VP_REDUCE_FMAX:
9700 case ISD::VP_REDUCE_FMAXIMUM:
9702 case ISD::VP_REDUCE_FMIN:
9703 case ISD::VP_REDUCE_FMINIMUM:
9713 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9718 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9719 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9720 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9721 "Unexpected reduction lowering");
9725 MVT ContainerVT = VecVT;
9734 VL =
Op.getOperand(3);
9736 std::tie(Mask, VL) =
9744 switch (
Op.getOpcode()) {
9748 case ISD::VP_REDUCE_AND: {
9758 case ISD::VP_REDUCE_OR:
9765 case ISD::VP_REDUCE_XOR: {
9788 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9792 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9793 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9794 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9795 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9811 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9815 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9818 if (M1VT != InnerVT)
9824 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9843 VecEVT =
Lo.getValueType();
9856 MVT ContainerVT = VecVT;
9876 Mask, VL,
DL, DAG, Subtarget);
9882static std::tuple<unsigned, SDValue, SDValue>
9886 auto Flags =
Op->getFlags();
9887 unsigned Opcode =
Op.getOpcode();
9911 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9919 MVT VecEltVT =
Op.getSimpleValueType();
9923 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9927 MVT ContainerVT = VecVT;
9933 MVT ResVT =
Op.getSimpleValueType();
9936 VL,
DL, DAG, Subtarget);
9941 if (
Op->getFlags().hasNoNaNs())
9947 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9948 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9954 DL, ResVT, NoNaNs, Res,
9962 unsigned Opc =
Op.getOpcode();
9985 Vec, Mask, VL,
DL, DAG, Subtarget);
9986 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
9987 Op->getFlags().hasNoNaNs())
10004 DL, ResVT, NoNaNs, Res,
10018 unsigned OrigIdx =
Op.getConstantOperandVal(2);
10027 (OrigIdx != 0 || !Vec.
isUndef())) {
10030 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10033 "Unexpected mask vector lowering");
10067 MVT ContainerVT = VecVT;
10075 DAG.
getUNDEF(ContainerVT), SubVec,
10082 DAG.
getUNDEF(ContainerVT), SubVec,
10098 if (OrigIdx == 0) {
10103 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
10104 SlideupAmt, Mask, VL, Policy);
10112 MVT ContainerVecVT = VecVT;
10118 MVT ContainerSubVecVT = SubVecVT;
10124 unsigned SubRegIdx;
10134 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10135 SubRegIdx = Decompose.first;
10137 (OrigIdx % Vscale));
10141 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
10142 SubRegIdx = Decompose.first;
10149 bool ExactlyVecRegSized =
10151 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
10166 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10170 if (SubRegIdx == RISCV::NoSubRegister) {
10189 MVT InterSubVT = ContainerVecVT;
10190 SDValue AlignedExtract = Vec;
10228 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10229 SlideupAmt, Mask, VL, Policy);
10234 if (ContainerVecVT.
bitsGT(InterSubVT))
10243 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10249 MVT SubVecVT =
Op.getSimpleValueType();
10254 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10265 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10268 "Unexpected mask vector lowering");
10307 MVT ContainerVT = VecVT;
10315 if (
auto ShrunkVT =
10317 ContainerVT = *ShrunkVT;
10331 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10343 MVT ContainerSubVecVT = SubVecVT;
10347 unsigned SubRegIdx;
10357 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10358 SubRegIdx = Decompose.first;
10360 (OrigIdx % Vscale));
10364 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10365 SubRegIdx = Decompose.first;
10388 MVT InterSubVT = VecVT;
10392 assert(SubRegIdx != RISCV::NoSubRegister);
10406 Vec, SlidedownAmt, Mask, VL);
10415 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10422 MVT VT =
N.getSimpleValueType();
10426 assert(
Op.getSimpleValueType() == VT &&
10427 "Operands and result must be same type");
10431 unsigned NumVals =
N->getNumValues();
10434 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10437 for (
unsigned I = 0;
I < NumVals;
I++) {
10443 if (TruncVals.
size() > 1)
10445 return TruncVals.
front();
10451 MVT VecVT =
Op.getSimpleValueType();
10454 "vector_interleave on non-scalable vector!");
10465 EVT SplitVT = Op0Lo.getValueType();
10468 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10470 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10484 Op.getOperand(0),
Op.getOperand(1));
10511 Concat, EvenIdx, Passthru, Mask, VL);
10513 Concat, OddIdx, Passthru, Mask, VL);
10527 MVT VecVT =
Op.getSimpleValueType();
10530 "vector_interleave on non-scalable vector!");
10543 EVT SplitVT = Op0Lo.getValueType();
10546 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10548 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10570 Op.getOperand(0),
Op.getOperand(1));
10618 MVT VT =
Op.getSimpleValueType();
10623 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10624 if (StepValImm != 1) {
10633 VL, VT,
DL, DAG, Subtarget);
10648 MVT VecVT =
Op.getSimpleValueType();
10658 unsigned MaxVLMAX =
10668 if (MaxVLMAX > 256 && EltSize == 8) {
10715 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10725 MVT VecVT =
Op.getSimpleValueType();
10729 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10730 SDValue DownOffset, UpOffset;
10731 if (ImmValue >= 0) {
10747 DownOffset, TrueMask, UpOffset);
10748 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10754RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10757 auto *
Load = cast<LoadSDNode>(
Op);
10760 Load->getMemoryVT(),
10761 *
Load->getMemOperand()) &&
10762 "Expecting a correctly-aligned load");
10764 MVT VT =
Op.getSimpleValueType();
10770 const auto [MinVLMAX, MaxVLMAX] =
10773 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10787 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10796 Load->getMemoryVT(),
Load->getMemOperand());
10803RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10806 auto *
Store = cast<StoreSDNode>(
Op);
10809 Store->getMemoryVT(),
10810 *
Store->getMemOperand()) &&
10811 "Expecting a correctly-aligned store");
10833 const auto [MinVLMAX, MaxVLMAX] =
10836 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10848 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10851 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10852 Store->getMemoryVT(),
Store->getMemOperand());
10858 MVT VT =
Op.getSimpleValueType();
10860 const auto *MemSD = cast<MemSDNode>(
Op);
10861 EVT MemVT = MemSD->getMemoryVT();
10863 SDValue Chain = MemSD->getChain();
10867 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10868 Mask = VPLoad->getMask();
10870 VL = VPLoad->getVectorLength();
10872 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10873 Mask = MLoad->getMask();
10874 PassThru = MLoad->getPassThru();
10881 MVT ContainerVT = VT;
10895 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10912 Chain =
Result.getValue(1);
10924 const auto *MemSD = cast<MemSDNode>(
Op);
10925 EVT MemVT = MemSD->getMemoryVT();
10927 SDValue Chain = MemSD->getChain();
10931 bool IsCompressingStore =
false;
10932 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10933 Val = VPStore->getValue();
10934 Mask = VPStore->getMask();
10935 VL = VPStore->getVectorLength();
10937 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10938 Val = MStore->getValue();
10939 Mask = MStore->getMask();
10940 IsCompressingStore = MStore->isCompressingStore();
10949 MVT ContainerVT = VT;
10954 if (!IsUnmasked || IsCompressingStore) {
10963 if (IsCompressingStore) {
10966 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10973 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10982 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10986RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10988 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10991 MVT VT =
Op.getSimpleValueType();
11005 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
11012 unsigned Opc =
Op.getOpcode();
11019 MVT VT =
Op.getSimpleValueType();
11052 MVT ContainerInVT = InVT;
11071 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11075 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11083 {Chain, Op1, Op2, CC, Mask, Mask, VL});
11088 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
11101 MVT VT =
Op.getSimpleValueType();
11105 "Unexpected type for ISD::ABS");
11107 MVT ContainerVT = VT;
11114 if (
Op->getOpcode() == ISD::VP_ABS) {
11115 Mask =
Op->getOperand(1);
11119 VL =
Op->getOperand(2);
11127 DAG.
getUNDEF(ContainerVT), Mask, VL);
11129 DAG.
getUNDEF(ContainerVT), Mask, VL);
11136SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
11139 MVT VT =
Op.getSimpleValueType();
11143 "Can only handle COPYSIGN with matching types.");
11152 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
11157SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
11159 MVT VT =
Op.getSimpleValueType();
11162 MVT I1ContainerVT =
11176 Op2, DAG.
getUNDEF(ContainerVT), VL);
11187 MVT VT =
Op.getSimpleValueType();
11192 for (
const SDValue &V :
Op->op_values()) {
11193 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11196 if (!
V.getValueType().isVector()) {
11202 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
11203 "Only fixed length vectors are supported!");
11217 if (
Op->isStrictFPOpcode()) {
11226 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11240 MVT VT =
Op.getSimpleValueType();
11243 MVT ContainerVT = VT;
11249 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11255 if (*MaskIdx == OpIdx.index())
11259 if (
Op.getOpcode() == ISD::VP_MERGE) {
11263 assert(
Op.getOpcode() == ISD::VP_SELECT);
11270 if (!
V.getValueType().isFixedLengthVector()) {
11275 MVT OpVT =
V.getSimpleValueType();
11277 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11278 "Only fixed length vectors are supported!");
11283 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11293 MVT VT =
Op.getSimpleValueType();
11299 MVT ContainerVT = VT;
11309 DAG.
getUNDEF(ContainerVT), Zero, VL);
11312 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11314 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11317 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11326 MVT VT =
Op.getSimpleValueType();
11330 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11334 MVT ContainerVT = VT;
11344 switch (Condition) {
11412 MVT DstVT =
Op.getSimpleValueType();
11413 MVT SrcVT = Src.getSimpleValueType();
11426 if (DstEltSize >= SrcEltSize) {
11435 if (SrcEltSize == 1) {
11446 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11447 }
else if (DstEltSize > (2 * SrcEltSize)) {
11451 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11457 "Wrong input/output vector types");
11460 if (DstEltSize > (2 * SrcEltSize)) {
11476 MVT InterimFVT = DstVT;
11477 if (SrcEltSize > (2 * DstEltSize)) {
11478 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11485 if (InterimFVT != DstVT) {
11491 "Wrong input/output vector types");
11495 if (DstEltSize == 1) {
11498 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11508 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11518 while (InterimIVT != DstVT) {
11530 MVT VT =
Op.getSimpleValueType();
11537RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11549 MVT VT =
Op.getSimpleValueType();
11550 MVT ContainerVT = VT;
11563 if (IsMaskVector) {
11574 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11583 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11586 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11587 SDValue DownOffset, UpOffset;
11588 if (ImmValue >= 0) {
11602 Op1, DownOffset, Mask, UpOffset);
11606 if (IsMaskVector) {
11610 {Result, DAG.getConstant(0, DL, ContainerVT),
11611 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11626 MVT VT =
Op.getSimpleValueType();
11628 MVT ContainerVT = VT;
11644RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11647 MVT VT =
Op.getSimpleValueType();
11654 MVT ContainerVT = VT;
11662 MVT GatherVT = ContainerVT;
11666 if (IsMaskVector) {
11677 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11683 unsigned MaxVLMAX =
11692 if (MaxVLMAX > 256 && EltSize == 8) {
11720 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11722 if (IsMaskVector) {
11745 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11747 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11749 DAG.
getUNDEF(GatherVT), Mask, EVL);
11751 if (IsMaskVector) {
11766 MVT VT =
Op.getSimpleValueType();
11768 return lowerVPOp(
Op, DAG);
11775 MVT ContainerVT = VT;
11794 MVT VT =
Op.getSimpleValueType();
11795 MVT ContainerVT = VT;
11801 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11807 : Intrinsic::riscv_vlse_mask,
11810 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11811 VPNode->getStride()};
11819 Ops.
push_back(VPNode->getVectorLength());
11827 VPNode->getMemoryVT(), VPNode->getMemOperand());
11841 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11842 SDValue StoreVal = VPNode->getValue();
11844 MVT ContainerVT = VT;
11855 : Intrinsic::riscv_vsse_mask,
11858 VPNode->getBasePtr(), VPNode->getStride()};
11866 Ops.
push_back(VPNode->getVectorLength());
11869 Ops, VPNode->getMemoryVT(),
11870 VPNode->getMemOperand());
11882 MVT VT =
Op.getSimpleValueType();
11884 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11885 EVT MemVT = MemSD->getMemoryVT();
11887 SDValue Chain = MemSD->getChain();
11893 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11894 Index = VPGN->getIndex();
11895 Mask = VPGN->getMask();
11897 VL = VPGN->getVectorLength();
11902 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11903 Index = MGN->getIndex();
11904 Mask = MGN->getMask();
11905 PassThru = MGN->getPassThru();
11909 MVT IndexVT =
Index.getSimpleValueType();
11913 "Unexpected VTs!");
11914 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11917 "Unexpected extending MGATHER/VP_GATHER");
11923 MVT ContainerVT = VT;
11947 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11964 Chain =
Result.getValue(1);
11981 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11982 EVT MemVT = MemSD->getMemoryVT();
11984 SDValue Chain = MemSD->getChain();
11987 [[maybe_unused]]
bool IsTruncatingStore =
false;
11990 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11991 Index = VPSN->getIndex();
11992 Mask = VPSN->getMask();
11993 Val = VPSN->getValue();
11994 VL = VPSN->getVectorLength();
11996 IsTruncatingStore =
false;
11999 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
12000 Index = MSN->getIndex();
12001 Mask = MSN->getMask();
12002 Val = MSN->getValue();
12003 IsTruncatingStore = MSN->isTruncatingStore();
12007 MVT IndexVT =
Index.getSimpleValueType();
12011 "Unexpected VTs!");
12012 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12015 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
12021 MVT ContainerVT = VT;
12045 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
12055 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
12064 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
12072 static const int Table =
12096 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
12102 static const unsigned Table =
12125 bool isRISCV64 = Subtarget.
is64Bit();
12189 switch (
N->getOpcode()) {
12191 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12197 "Unexpected custom legalisation");
12198 bool IsStrict =
N->isStrictFPOpcode();
12201 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12219 Opc,
DL, VTs, Chain, Op0,
12253 std::tie(Result, Chain) =
12254 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12282 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12293 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12294 "has custom type legalization on riscv32");
12296 SDValue LoCounter, HiCounter;
12300 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12302 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12305 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12307 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12311 N->getOperand(0), LoCounter, HiCounter);
12335 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12336 unsigned XLen = Subtarget.
getXLen();
12339 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12347 if (LHSIsU == RHSIsU)
12364 if (RHSIsU && LHSIsS && !RHSIsS)
12366 else if (LHSIsU && RHSIsS && !LHSIsS)
12376 "Unexpected custom legalisation");
12383 "Unexpected custom legalisation");
12386 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12412 "Unexpected custom legalisation");
12413 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12414 Subtarget.hasVendorXTHeadBb()) &&
12415 "Unexpected custom legalization");
12416 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12417 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12426 "Unexpected custom legalisation");
12440 MVT VT =
N->getSimpleValueType(0);
12441 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12442 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12443 "Unexpected custom legalisation");
12455 if (VT != MVT::i32)
12464 "Unexpected custom legalisation");
12468 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12485 EVT OType =
N->getValueType(1);
12498 "Unexpected custom legalisation");
12515 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12519 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12537 "Unexpected custom legalisation");
12538 if (Subtarget.hasStdExtZbb()) {
12559 "Unexpected custom legalisation");
12565 "Unexpected custom legalisation");
12567 if (Subtarget.hasStdExtZbb()) {
12601 EVT VT =
N->getValueType(0);
12606 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12610 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12611 Subtarget.hasStdExtZfbfmin()) {
12614 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12619 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12621 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12642 MVT VT =
N->getSimpleValueType(0);
12644 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12645 "Unexpected custom legalisation");
12648 "Unexpected extension");
12674 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12676 "Unexpected EXTRACT_VECTOR_ELT legalization");
12679 MVT ContainerVT = VecVT;
12707 DAG.
getUNDEF(ContainerVT), Mask, VL);
12715 unsigned IntNo =
N->getConstantOperandVal(0);
12719 "Don't know how to custom type legalize this intrinsic!");
12720 case Intrinsic::experimental_get_vector_length: {
12725 case Intrinsic::experimental_cttz_elts: {
12731 case Intrinsic::riscv_orc_b:
12732 case Intrinsic::riscv_brev8:
12733 case Intrinsic::riscv_sha256sig0:
12734 case Intrinsic::riscv_sha256sig1:
12735 case Intrinsic::riscv_sha256sum0:
12736 case Intrinsic::riscv_sha256sum1:
12737 case Intrinsic::riscv_sm3p0:
12738 case Intrinsic::riscv_sm3p1: {
12739 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12759 case Intrinsic::riscv_sm4ks:
12760 case Intrinsic::riscv_sm4ed: {
12768 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12772 case Intrinsic::riscv_mopr: {
12773 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12783 case Intrinsic::riscv_moprr: {
12784 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12796 case Intrinsic::riscv_clmul: {
12797 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12808 case Intrinsic::riscv_clmulh:
12809 case Intrinsic::riscv_clmulr: {
12810 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12838 case Intrinsic::riscv_vmv_x_s: {
12839 EVT VT =
N->getValueType(0);
12841 if (VT.
bitsLT(XLenVT)) {
12850 "Unexpected custom legalization");
12888 case ISD::VP_REDUCE_ADD:
12889 case ISD::VP_REDUCE_AND:
12890 case ISD::VP_REDUCE_OR:
12891 case ISD::VP_REDUCE_XOR:
12892 case ISD::VP_REDUCE_SMAX:
12893 case ISD::VP_REDUCE_UMAX:
12894 case ISD::VP_REDUCE_SMIN:
12895 case ISD::VP_REDUCE_UMIN:
12959 const EVT VT =
N->getValueType(0);
12960 const unsigned Opc =
N->getOpcode();
12967 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12972 "Inconsistent mappings");
12983 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12986 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
13001 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
13003 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
13004 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
13008 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
13015 if (
LHS.getOpcode() != ReduceOpc)
13030 auto Flags = ReduceVec->
getFlags();
13031 Flags.intersectWith(
N->getFlags());
13032 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
13042 auto BinOpToRVVReduce = [](
unsigned Opc) {
13071 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
13074 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
13077 unsigned Opc =
N->getOpcode();
13078 unsigned ReduceIdx;
13079 if (IsReduction(
N->getOperand(0), Opc))
13081 else if (IsReduction(
N->getOperand(1), Opc))
13087 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
13090 SDValue Extract =
N->getOperand(ReduceIdx);
13122 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
13149 if (!Subtarget.hasStdExtZba())
13153 EVT VT =
N->getValueType(0);
13165 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13166 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
13169 int64_t C0 = N0C->getSExtValue();
13170 int64_t C1 = N1C->getSExtValue();
13171 if (C0 <= 0 || C1 <= 0)
13175 int64_t Bits = std::min(C0, C1);
13176 int64_t Diff = std::abs(C0 - C1);
13177 if (Diff != 1 && Diff != 2 && Diff != 3)
13204 EVT VT =
N->getValueType(0);
13212 if ((!Subtarget.hasStdExtZicond() &&
13213 !Subtarget.hasVendorXVentanaCondOps()) ||
13235 bool SwapSelectOps;
13241 SwapSelectOps =
false;
13242 NonConstantVal = FalseVal;
13244 SwapSelectOps =
true;
13245 NonConstantVal = TrueVal;
13251 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13299 EVT VT =
N->getValueType(0);
13307 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13308 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13314 if (!N0C->hasOneUse())
13316 int64_t C0 = N0C->getSExtValue();
13317 int64_t C1 = N1C->getSExtValue();
13319 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13322 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
13323 !isInt<12>(C0 * (C1 / C0))) {
13326 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
13327 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13330 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13331 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13358 EVT VT =
N->getValueType(0);
13389 unsigned OuterExtend =
13393 OuterExtend,
SDLoc(
N), VT,
13401 EVT VT =
N->getValueType(0);
13449 EVT VT =
N->getValueType(0);
13453 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13459 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13469 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13490 if (!Subtarget.hasStdExtZbb())
13493 EVT VT =
N->getValueType(0);
13495 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
13504 auto *ShAmtC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
13505 if (!ShAmtC || ShAmtC->getZExtValue() != 8)
13520 EVT VT =
N->getValueType(0);
13551 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13575 EVT VT =
N->getValueType(0);
13599 EVT VT =
N->getValueType(0);
13626 if (CondLHS != True)
13633 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
13645 if (!FalseRHSC || !FalseRHSC->
isZero())
13665 EVT VT =
N->getValueType(0);
13751 EVT VT =
N->getValueType(0);
13822 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13827 const APInt &Imm = ConstN00->getAPIntValue();
13828 if ((Imm + 1).isSignedIntN(12))
13845 LHS.getValueType());
13866 EVT VT =
N->getValueType(0);
13878 const bool HasShlAdd =
13879 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
13893 for (
uint64_t Divisor : {3, 5, 9}) {
13894 if (MulAmt % Divisor != 0)
13896 uint64_t MulAmt2 = MulAmt / Divisor;
13903 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
13904 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
13921 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13937 if (ScaleShift >= 1 && ScaleShift < 4) {
13938 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13952 for (
uint64_t Divisor : {3, 5, 9}) {
13957 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13968 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13970 if (ScaleShift >= 1 && ScaleShift < 4) {
13971 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13997 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
13999 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
14016 EVT VT =
N->getValueType(0);
14023 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
14024 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
14037 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
14038 V3 != (HalfSize - 1))
14054 EVT VT =
N->getValueType(0);
14062 unsigned AddSubOpc;
14068 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
14069 AddSubOpc = V->getOpcode();
14071 SDValue Opnd = V->getOperand(1);
14072 MulOper = V->getOperand(0);
14081 if (IsAddSubWith1(N0)) {
14083 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
14086 if (IsAddSubWith1(N1)) {
14088 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
14103 if (isIndexTypeSigned(IndexType))
14106 if (!
N->hasOneUse())
14109 EVT VT =
N.getValueType();
14148 EVT SrcVT = Src.getValueType();
14152 NewElen = std::max(NewElen, 8U);
14175 EVT VT =
N->getValueType(0);
14178 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
14182 auto *N1C = dyn_cast<ConstantSDNode>(N1);
14194 if (!isIntEqualitySetCC(
Cond))
14203 const APInt &C1 = N1C->getAPIntValue();
14221 EVT VT =
N->getValueType(0);
14225 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
14227 Src.getOperand(0));
14235struct CombineResult;
14237enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
14264struct NodeExtensionHelper {
14273 bool SupportsFPExt;
14276 bool EnforceOneUse;
14291 return OrigOperand;
14302 unsigned getExtOpc(ExtKind SupportsExt)
const {
14303 switch (SupportsExt) {
14304 case ExtKind::SExt:
14306 case ExtKind::ZExt:
14308 case ExtKind::FPExt:
14319 std::optional<ExtKind> SupportsExt)
const {
14320 if (!SupportsExt.has_value())
14321 return OrigOperand;
14323 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
14327 if (
Source.getValueType() == NarrowVT)
14330 unsigned ExtOpc = getExtOpc(*SupportsExt);
14334 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
14341 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
14353 DAG.
getUNDEF(NarrowVT), Source, VL);
14366 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
14372 MVT EltVT = SupportsExt == ExtKind::FPExt
14374 :
MVT::getIntegerVT(NarrowSize);
14376 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
14377 "Trying to extend something we can't represent");
14384 static unsigned getSExtOpcode(
unsigned Opcode) {
14407 static unsigned getZExtOpcode(
unsigned Opcode) {
14433 static unsigned getFPExtOpcode(
unsigned Opcode) {
14450 static unsigned getSUOpcode(
unsigned Opcode) {
14452 "SU is only supported for MUL");
14458 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14478 using CombineToTry = std::function<std::optional<CombineResult>(
14479 SDNode * ,
const NodeExtensionHelper & ,
14484 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14488 unsigned Opc = OrigOperand.
getOpcode();
14492 "Unexpected Opcode");
14505 unsigned ScalarBits =
Op.getValueSizeInBits();
14507 if (ScalarBits < EltBits) {
14510 !Subtarget.
is64Bit() &&
"Unexpected splat");
14512 SupportsSExt =
true;
14516 SupportsZExt =
true;
14518 EnforceOneUse =
false;
14522 unsigned NarrowSize = EltBits / 2;
14525 if (NarrowSize < 8)
14529 SupportsSExt =
true;
14533 SupportsZExt =
true;
14535 EnforceOneUse =
false;
14542 SupportsZExt =
false;
14543 SupportsSExt =
false;
14544 SupportsFPExt =
false;
14545 EnforceOneUse =
true;
14546 unsigned Opc = OrigOperand.
getOpcode();
14568 SupportsZExt =
true;
14571 SupportsSExt =
true;
14574 SupportsFPExt =
true;
14578 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14591 unsigned ScalarBits =
Op.getOperand(0).getValueSizeInBits();
14592 if (NarrowSize != ScalarBits)
14595 SupportsFPExt =
true;
14604 static bool isSupportedRoot(
const SDNode *Root,
14633 Subtarget.hasStdExtZvbb();
14635 return Subtarget.hasStdExtZvbb();
14644 assert(isSupportedRoot(Root, Subtarget) &&
14645 "Trying to build an helper with an "
14646 "unsupported root");
14647 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14663 if (OperandIdx == 1) {
14672 EnforceOneUse =
false;
14677 fillUpExtensionSupport(Root, DAG, Subtarget);
14683 static std::pair<SDValue, SDValue>
14686 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14705 switch (
N->getOpcode()) {
14743struct CombineResult {
14745 unsigned TargetOpcode;
14747 std::optional<ExtKind> LHSExt;
14748 std::optional<ExtKind> RHSExt;
14752 NodeExtensionHelper
LHS;
14754 NodeExtensionHelper
RHS;
14756 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14757 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14758 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14759 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14768 std::tie(Mask, VL) =
14769 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14783 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14784 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14799static std::optional<CombineResult>
14800canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14801 const NodeExtensionHelper &RHS,
14804 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14805 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14806 Root, LHS, {ExtKind::ZExt}, RHS,
14808 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14809 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14810 Root, LHS, {ExtKind::SExt}, RHS,
14812 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14813 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14814 Root, LHS, {ExtKind::FPExt}, RHS,
14816 return std::nullopt;
14825static std::optional<CombineResult>
14826canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14829 return canFoldToVWWithSameExtensionImpl(
14830 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14838static std::optional<CombineResult>
14839canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14842 if (
RHS.SupportsFPExt)
14843 return CombineResult(
14844 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14845 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14852 return CombineResult(
14853 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14854 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14856 return CombineResult(
14857 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14858 LHS, std::nullopt, RHS, {ExtKind::SExt});
14859 return std::nullopt;
14866static std::optional<CombineResult>
14867canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14870 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14878static std::optional<CombineResult>
14879canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14882 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14890static std::optional<CombineResult>
14891canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14894 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14902static std::optional<CombineResult>
14903canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14907 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14908 return std::nullopt;
14909 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14910 Root, LHS, {ExtKind::SExt}, RHS,
14915NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14926 Strategies.
push_back(canFoldToVWWithSameExtension);
14931 Strategies.
push_back(canFoldToVWWithSameExtension);
14936 Strategies.
push_back(canFoldToVWWithSameExtension);
14943 Strategies.
push_back(canFoldToVWWithZEXT);
14948 Strategies.
push_back(canFoldToVWWithSEXT);
14953 Strategies.
push_back(canFoldToVWWithZEXT);
14958 Strategies.
push_back(canFoldToVWWithFPEXT);
14987 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14993 Inserted.insert(
N);
14996 while (!Worklist.
empty()) {
14998 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
15001 NodeExtensionHelper
LHS(Root, 0, DAG, Subtarget);
15002 NodeExtensionHelper
RHS(Root, 1, DAG, Subtarget);
15003 auto AppendUsersIfNeeded = [&Worklist,
15004 &Inserted](
const NodeExtensionHelper &
Op) {
15005 if (
Op.needToPromoteOtherUsers()) {
15006 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
15007 if (Inserted.insert(TheUse).second)
15019 NodeExtensionHelper::getSupportedFoldings(Root);
15021 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
15022 bool Matched =
false;
15023 for (
int Attempt = 0;
15024 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
15027 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
15028 FoldingStrategies) {
15029 std::optional<CombineResult> Res =
15030 FoldingStrategy(Root,
LHS,
RHS, DAG, Subtarget);
15037 if (Res->LHSExt.has_value())
15038 AppendUsersIfNeeded(
LHS);
15039 if (Res->RHSExt.has_value())
15040 AppendUsersIfNeeded(
RHS);
15051 SDValue InputRootReplacement;
15058 for (CombineResult Res : CombinesToApply) {
15059 SDValue NewValue = Res.materialize(DAG, Subtarget);
15060 if (!InputRootReplacement) {
15062 "First element is expected to be the current node");
15063 InputRootReplacement = NewValue;
15068 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
15072 return InputRootReplacement;
15079 unsigned Opc =
N->getOpcode();
15084 SDValue MergeOp =
N->getOperand(1);
15085 unsigned MergeOpc = MergeOp.
getOpcode();
15096 SDValue Passthru =
N->getOperand(2);
15110 Z = Z.getOperand(1);
15116 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
15123 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
15152 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
15158 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
15160 if (MemVT == MVT::i32)
15166 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
15201 if (!Subtarget.hasVendorXTHeadMemPair())
15213 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
15215 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
15216 return {
Ptr->getOperand(0), C1->getZExtValue()};
15220 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
15243 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
15246 if (Base1 != Base2)
15250 bool Valid =
false;
15251 if (MemVT == MVT::i32) {
15253 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
15255 }
else if (MemVT == MVT::i64) {
15257 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
15291 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15299 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15309 EVT VT =
N->getValueType(0);
15312 MVT SrcVT = Src.getSimpleValueType();
15313 MVT SrcContainerVT = SrcVT;
15315 SDValue XVal = Src.getOperand(0);
15342 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15346 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15350 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
15363 if (VT != MVT::i32 && VT != XLenVT)
15393 EVT DstVT =
N->getValueType(0);
15394 if (DstVT != XLenVT)
15400 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15408 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15411 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
15420 if (SatVT == DstVT)
15422 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
15428 Src = Src.getOperand(0);
15449 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
15455 EVT VT =
N->getValueType(0);
15508 unsigned Offset =
N->isTargetStrictFPOpcode();
15515 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15517 V.getOperand(2) == VL) {
15519 V = V.getOperand(0);
15526 bool NegA = invertIfNegative(
A);
15527 bool NegB = invertIfNegative(
B);
15528 bool NegC = invertIfNegative(
C);
15531 if (!NegA && !NegB && !NegC)
15535 if (
N->isTargetStrictFPOpcode())
15537 {N->getOperand(0), A, B, C, Mask, VL});
15547 if (
N->getValueType(0).getVectorElementType() == MVT::f32 &&
15552 if (
N->isTargetStrictFPOpcode())
15577 switch (
N->getOpcode()) {
15597 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15598 N->getOperand(2), Mask, VL);
15605 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15608 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15610 uint64_t ShAmt =
N->getConstantOperandVal(1);
15621 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15649 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15662 !isa<ConstantSDNode>(U->getOperand(1)) ||
15663 U->getConstantOperandVal(1) > 32)
15718 if (!
Cond.hasOneUse())
15737 EVT VT =
Cond.getValueType();
15782 LHS =
LHS.getOperand(0);
15792 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15796 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15800 RHS =
LHS.getOperand(1);
15801 LHS =
LHS.getOperand(0);
15810 RHS =
LHS.getOperand(1);
15811 LHS =
LHS.getOperand(0);
15827 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15868 bool Commutative =
true;
15869 unsigned Opc = TrueVal.getOpcode();
15877 Commutative =
false;
15885 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15889 if (FalseVal == TrueVal.getOperand(0))
15891 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15896 EVT VT =
N->getValueType(0);
15898 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15904 assert(IdentityOperand &&
"No identity operand!");
15909 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15910 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15931 CountZeroes =
N->getOperand(2);
15932 ValOnZero =
N->getOperand(1);
15934 CountZeroes =
N->getOperand(1);
15935 ValOnZero =
N->getOperand(2);
15954 if (
Cond->getOperand(0) != CountZeroesArgument)
15970 CountZeroes, BitWidthMinusOne);
15980 EVT VT =
N->getValueType(0);
15981 EVT CondVT =
Cond.getValueType();
15989 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15995 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
16016 SDValue TrueVal =
N->getOperand(1);
16017 SDValue FalseVal =
N->getOperand(2);
16032 EVT VT =
N->getValueType(0);
16039 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
16054 if (
Op.isUndef()) {
16067 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
16071 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
16072 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
16076 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
16104 const unsigned InVecOpcode = InVec->
getOpcode();
16114 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
16121 InVecLHS, InValLHS, EltNo);
16123 InVecRHS, InValRHS, EltNo);
16132 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
16135 unsigned Elt = IndexC->getZExtValue();
16143 unsigned ConcatOpIdx = Elt / ConcatNumElts;
16146 ConcatOp, InVal, NewIdx);
16150 ConcatOps[ConcatOpIdx] = ConcatOp;
16162 EVT VT =
N->getValueType(0);
16172 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
16174 !
SDValue(BaseLd, 0).hasOneUse())
16177 EVT BaseLdVT = BaseLd->getValueType(0);
16184 auto *Ld = dyn_cast<LoadSDNode>(
Op);
16185 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
16187 Ld->getValueType(0) != BaseLdVT)
16196 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
16198 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
16203 if (BIO1.equalBaseIndex(BIO2, DAG))
16208 SDValue P2 = Ld2->getBasePtr();
16214 return std::nullopt;
16218 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
16223 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
16224 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
16232 unsigned WideScalarBitWidth =
16245 auto [StrideVariant, MustNegateStride] = *BaseDiff;
16246 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
16247 ? std::get<SDValue>(StrideVariant)
16250 if (MustNegateStride)
16258 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
16259 ConstStride && ConstStride->getSExtValue() >= 0)
16263 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
16269 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
16273 WideVecVT,
DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
16288 if (
N->getValueType(0).isFixedLengthVector())
16291 SDValue Addend =
N->getOperand(0);
16295 SDValue AddMergeOp =
N->getOperand(2);
16300 auto IsVWMulOpc = [](
unsigned Opc) {
16329 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
16330 }(
N, DAG, Subtarget);
16335 if (AddMask != MulMask || AddVL != MulVL)
16340 "Unexpected opcode after VWMACC_VL");
16342 "Unexpected opcode after VWMACC_VL!");
16344 "Unexpected opcode after VWMUL_VL!");
16346 "Unexpected opcode after VWMUL_VL!");
16349 EVT VT =
N->getValueType(0);
16365 const EVT IndexVT =
Index.getValueType();
16369 if (!isIndexTypeSigned(IndexType))
16401 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16404 if (
Index->getOperand(i)->isUndef())
16407 if (
C % ElementSize != 0)
16409 C =
C / ElementSize;
16413 ActiveLanes.
set(
C);
16415 return ActiveLanes.
all();
16433 if (NumElems % 2 != 0)
16437 const unsigned WiderElementSize = ElementSize * 2;
16438 if (WiderElementSize > ST.getELen()/8)
16441 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
16444 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16447 if (
Index->getOperand(i)->isUndef())
16453 if (
C % WiderElementSize != 0)
16458 if (
C !=
Last + ElementSize)
16475 (isa<RegisterSDNode>(VL) &&
16476 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16478 Mask.getOperand(0) != VL)
16481 auto IsTruncNode = [&](
SDValue V) {
16483 V.getOperand(1) == Mask && V.getOperand(2) == VL;
16490 while (IsTruncNode(
Op)) {
16491 if (!
Op.hasOneUse())
16493 Op =
Op.getOperand(0);
16528 MVT VT =
N->getSimpleValueType(0);
16533 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
16535 if (V.getOpcode() != Opc &&
16536 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
16537 V.getOperand(3) == Mask && V.getOperand(4) == VL))
16545 Op.getOperand(1).getValueType().isFixedLengthVector() &&
16547 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
16549 Op =
Op.getOperand(1).getOperand(0);
16552 return V.getOperand(0);
16555 Op.getOperand(2) == VL) {
16556 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
16558 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
16559 return V.getOperand(0);
16568 auto DetectUSatPattern = [&](
SDValue V) {
16593 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
16599 auto DetectSSatPattern = [&](
SDValue V) {
16601 unsigned NumSrcBits = V.getScalarValueSizeInBits();
16609 if (HiC == SignedMax && LoC == SignedMin)
16615 if (HiC == SignedMax && LoC == SignedMin)
16625 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
16627 Src = Src.getOperand(0);
16631 if ((Val = DetectUSatPattern(Src)))
16633 else if ((Val = DetectSSatPattern(Src)))
16645 ClipOpc,
DL, ValVT,
16649 }
while (ValVT != VT);
16663 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16674 switch (
N->getOpcode()) {
16694 APInt V =
C->getValueAPF().bitcastToAPInt();
16729 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16730 SimplifyDemandedLowBitsHelper(1, 5))
16738 if (SimplifyDemandedLowBitsHelper(0, 32))
16755 MVT VT =
N->getSimpleValueType(0);
16764 "Unexpected value type!");
16787 EVT VT =
N->getValueType(0);
16852 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16857 Src.getOperand(0));
16862 Src.getOperand(0), Src.getOperand(1));
16883 unsigned Opc =
N->getOpcode();
16898 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16908 N->getValueType(0), Val,
Cond.getOperand(0));
16919 SDValue FalseV =
N->getOperand(4);
16921 EVT VT =
N->getValueType(0);
16924 if (TrueV == FalseV)
16929 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16935 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16936 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16939 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16940 isInt<12>(TrueSImm - FalseSImm)) {
16956 {LHS, RHS, CC, TrueV, FalseV});
17023 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
17036 EVT VT =
N->getValueType(0);
17060 const auto *MGN = cast<MaskedGatherSDNode>(
N);
17061 const EVT VT =
N->getValueType(0);
17063 SDValue ScaleOp = MGN->getScale();
17065 assert(!MGN->isIndexScaled() &&
17066 "Scaled gather/scatter should not be formed");
17071 N->getVTList(), MGN->getMemoryVT(),
DL,
17072 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17073 MGN->getBasePtr(), Index, ScaleOp},
17074 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17078 N->getVTList(), MGN->getMemoryVT(),
DL,
17079 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17080 MGN->getBasePtr(), Index, ScaleOp},
17081 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17087 if (std::optional<VIDSequence> SimpleVID =
17089 SimpleVID && SimpleVID->StepDenominator == 1) {
17090 const int64_t StepNumerator = SimpleVID->StepNumerator;
17091 const int64_t Addend = SimpleVID->Addend;
17098 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
17108 MGN->getMask(), EVL, MGN->getMemOperand());
17110 StridedLoad, MGN->getPassThru(), EVL);
17120 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
17122 MGN->getMemoryVT(), MGN->getMemOperand(),
17131 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
17133 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
17135 EVT IndexVT =
Index.getValueType()
17142 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
17144 EltCnt.divideCoefficientBy(2));
17147 EltCnt.divideCoefficientBy(2));
17152 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
17161 const auto *MSN = cast<MaskedScatterSDNode>(
N);
17163 SDValue ScaleOp = MSN->getScale();
17165 assert(!MSN->isIndexScaled() &&
17166 "Scaled gather/scatter should not be formed");
17171 N->getVTList(), MSN->getMemoryVT(),
DL,
17172 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17174 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17178 N->getVTList(), MSN->getMemoryVT(),
DL,
17179 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17181 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17183 EVT VT = MSN->getValue()->getValueType(0);
17185 if (!MSN->isTruncatingStore() &&
17189 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
17190 DAG.
getUNDEF(XLenVT), MSN->getMask(),
17191 MSN->getMemoryVT(), MSN->getMemOperand(),
17196 case ISD::VP_GATHER: {
17197 const auto *VPGN = cast<VPGatherSDNode>(
N);
17199 SDValue ScaleOp = VPGN->getScale();
17201 assert(!VPGN->isIndexScaled() &&
17202 "Scaled gather/scatter should not be formed");
17207 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17208 ScaleOp, VPGN->getMask(),
17209 VPGN->getVectorLength()},
17210 VPGN->getMemOperand(), IndexType);
17214 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17215 ScaleOp, VPGN->getMask(),
17216 VPGN->getVectorLength()},
17217 VPGN->getMemOperand(), IndexType);
17221 case ISD::VP_SCATTER: {
17222 const auto *VPSN = cast<VPScatterSDNode>(
N);
17224 SDValue ScaleOp = VPSN->getScale();
17226 assert(!VPSN->isIndexScaled() &&
17227 "Scaled gather/scatter should not be formed");
17232 {VPSN->getChain(), VPSN->getValue(),
17233 VPSN->getBasePtr(), Index, ScaleOp,
17234 VPSN->getMask(), VPSN->getVectorLength()},
17235 VPSN->getMemOperand(), IndexType);
17239 {VPSN->getChain(), VPSN->getValue(),
17240 VPSN->getBasePtr(), Index, ScaleOp,
17241 VPSN->getMask(), VPSN->getVectorLength()},
17242 VPSN->getMemOperand(), IndexType);
17256 EVT VT =
N->getValueType(0);
17259 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
17260 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
17278 EVT VT =
N->getValueType(0);
17282 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
17312 if (
N->getValueType(0).getVectorElementType() == MVT::f32 &&
17326 auto *Store = cast<StoreSDNode>(
N);
17327 SDValue Chain = Store->getChain();
17328 EVT MemVT = Store->getMemoryVT();
17329 SDValue Val = Store->getValue();
17332 bool IsScalarizable =
17334 Store->isSimple() &&
17364 NewVT, *Store->getMemOperand())) {
17366 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
17367 Store->getPointerInfo(), Store->getOriginalAlign(),
17368 Store->getMemOperand()->getFlags());
17376 if (
auto *L = dyn_cast<LoadSDNode>(Val);
17378 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
17380 L->getMemoryVT() == MemVT) {
17383 NewVT, *Store->getMemOperand()) &&
17385 NewVT, *L->getMemOperand())) {
17387 L->getPointerInfo(), L->getOriginalAlign(),
17388 L->getMemOperand()->getFlags());
17389 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
17390 Store->getPointerInfo(), Store->getOriginalAlign(),
17391 Store->getMemOperand()->getFlags());
17403 MVT VecVT = Src.getSimpleValueType();
17410 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
17413 Store->getMemOperand(), Store->getAddressingMode(),
17414 Store->isTruncatingStore(),
false);
17421 EVT VT =
N->getValueType(0);
17443 const MVT VT =
N->getSimpleValueType(0);
17444 SDValue Passthru =
N->getOperand(0);
17445 SDValue Scalar =
N->getOperand(1);
17454 const MVT VT =
N->getSimpleValueType(0);
17455 SDValue Passthru =
N->getOperand(0);
17456 SDValue Scalar =
N->getOperand(1);
17461 unsigned ScalarSize = Scalar.getValueSizeInBits();
17463 if (ScalarSize > EltWidth && Passthru.
isUndef())
17464 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
17471 (!Const || Const->isZero() ||
17472 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
17482 if (
N->getOperand(0).isUndef() &&
17485 Src.getOperand(0).getValueType().isScalableVector()) {
17486 EVT VT =
N->getValueType(0);
17487 EVT SrcVT = Src.getOperand(0).getValueType();
17491 return Src.getOperand(0);
17497 const MVT VT =
N->getSimpleValueType(0);
17498 SDValue Passthru =
N->getOperand(0);
17499 SDValue Scalar =
N->getOperand(1);
17503 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
17504 return Scalar.getOperand(0);
17513 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
17523 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
17531 MVT VecVT =
N->getOperand(0).getSimpleValueType();
17533 if (M1VT.
bitsLT(VecVT)) {
17544 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
17549 case Intrinsic::riscv_masked_strided_load: {
17550 MVT VT =
N->getSimpleValueType(0);
17551 auto *Load = cast<MemIntrinsicSDNode>(
N);
17552 SDValue PassThru =
N->getOperand(2);
17554 SDValue Stride =
N->getOperand(4);
17560 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
17561 StrideC && StrideC->getZExtValue() == ElementSize)
17563 DAG.
getUNDEF(XLenVT), Mask, PassThru,
17564 Load->getMemoryVT(), Load->getMemOperand(),
17568 case Intrinsic::riscv_masked_strided_store: {
17569 auto *Store = cast<MemIntrinsicSDNode>(
N);
17572 SDValue Stride =
N->getOperand(4);
17577 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
17578 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
17579 StrideC && StrideC->getZExtValue() == ElementSize)
17582 Value.getValueType(), Store->getMemOperand(),
17586 case Intrinsic::riscv_vcpop:
17587 case Intrinsic::riscv_vcpop_mask:
17588 case Intrinsic::riscv_vfirst:
17589 case Intrinsic::riscv_vfirst_mask: {
17591 if (IntNo == Intrinsic::riscv_vcpop_mask ||
17592 IntNo == Intrinsic::riscv_vfirst_mask)
17593 VL =
N->getOperand(3);
17598 EVT VT =
N->getValueType(0);
17599 if (IntNo == Intrinsic::riscv_vfirst ||
17600 IntNo == Intrinsic::riscv_vfirst_mask)
17609 EVT VT =
N->getValueType(0);
17613 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17632 EVT XVT,
unsigned KeptBits)
const {
17637 if (XVT != MVT::i32 && XVT != MVT::i64)
17641 if (KeptBits == 32 || KeptBits == 64)
17645 return Subtarget.hasStdExtZbb() &&
17646 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17654 "Expected shift op");
17665 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
17666 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17668 const APInt &C1Int = C1->getAPIntValue();
17669 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17695 if (C1Cost < ShiftedC1Cost)
17709 EVT VT =
Op.getValueType();
17713 unsigned Opcode =
Op.getOpcode();
17721 const APInt &Mask =
C->getAPIntValue();
17730 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17731 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17733 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17734 if (NewMask == Mask)
17739 Op.getOperand(0), NewC);
17752 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17753 if (IsLegalMask(NewMask))
17754 return UseMask(NewMask);
17757 if (VT == MVT::i64) {
17759 if (IsLegalMask(NewMask))
17760 return UseMask(NewMask);
17775 APInt NewMask = ShrunkMask;
17776 if (MinSignedBits <= 12)
17778 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17784 assert(IsLegalMask(NewMask));
17785 return UseMask(NewMask);
17789 static const uint64_t GREVMasks[] = {
17790 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17791 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17793 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17794 unsigned Shift = 1 << Stage;
17795 if (ShAmt & Shift) {
17797 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17809 const APInt &DemandedElts,
17811 unsigned Depth)
const {
17813 unsigned Opc =
Op.getOpcode();
17818 "Should use MaskedValueIsZero if you don't know whether Op"
17819 " is a target node!");
17902 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17905 if (MaxVLenB == MinVLenB)
17922 case Intrinsic::riscv_vsetvli:
17923 case Intrinsic::riscv_vsetvlimax: {
17924 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17925 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17931 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17934 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17935 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17937 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17950 unsigned Depth)
const {
17951 switch (
Op.getOpcode()) {
17957 if (Tmp == 1)
return 1;
17960 return std::min(Tmp, Tmp2);
17972 if (Tmp < 33)
return 1;
17997 unsigned XLen = Subtarget.
getXLen();
17998 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17999 if (EltBits <= XLen)
18000 return XLen - EltBits + 1;
18004 unsigned IntNo =
Op.getConstantOperandVal(1);
18008 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
18009 case Intrinsic::riscv_masked_atomicrmw_add_i64:
18010 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
18011 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
18012 case Intrinsic::riscv_masked_atomicrmw_max_i64:
18013 case Intrinsic::riscv_masked_atomicrmw_min_i64:
18014 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
18015 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
18016 case Intrinsic::riscv_masked_cmpxchg_i64:
18024 assert(Subtarget.hasStdExtA());
18039 switch (
Op.getOpcode()) {
18045 return !
Op.getValueType().isInteger();
18053 assert(Ld &&
"Unexpected null LoadSDNode");
18061 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
18062 if (!CNode || CNode->isMachineConstantPoolEntry() ||
18063 CNode->getOffset() != 0)
18071 auto *CNode = GetSupportedConstantPool(
Ptr);
18072 if (!CNode || CNode->getTargetFlags() != 0)
18075 return CNode->getConstVal();
18083 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
18084 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
18090 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
18093 return CNodeLo->getConstVal();
18098 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
18130 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18133 int64_t LoCounter =
MI.getOperand(2).getImm();
18134 int64_t HiCounter =
MI.getOperand(3).getImm();
18144 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
18156 MI.eraseFromParent();
18164 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
18172 Register SrcReg =
MI.getOperand(2).getReg();
18192 MI.eraseFromParent();
18199 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
18200 "Unexpected instruction");
18206 Register DstReg =
MI.getOperand(0).getReg();
18229 MI.eraseFromParent();
18234 switch (
MI.getOpcode()) {
18237 case RISCV::Select_GPR_Using_CC_GPR:
18238 case RISCV::Select_GPR_Using_CC_Imm:
18239 case RISCV::Select_FPR16_Using_CC_GPR:
18240 case RISCV::Select_FPR16INX_Using_CC_GPR:
18241 case RISCV::Select_FPR32_Using_CC_GPR:
18242 case RISCV::Select_FPR32INX_Using_CC_GPR:
18243 case RISCV::Select_FPR64_Using_CC_GPR:
18244 case RISCV::Select_FPR64INX_Using_CC_GPR:
18245 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18251 unsigned RelOpcode,
unsigned EqOpcode,
18254 Register DstReg =
MI.getOperand(0).getReg();
18255 Register Src1Reg =
MI.getOperand(1).getReg();
18256 Register Src2Reg =
MI.getOperand(2).getReg();
18258 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18282 MI.eraseFromParent();
18333 F->insert(It, FirstMBB);
18334 F->insert(It, SecondMBB);
18335 F->insert(It, SinkMBB);
18384 First.eraseFromParent();
18423 if (
MI.getOperand(2).isReg())
18424 RHS =
MI.getOperand(2).getReg();
18429 SelectDests.
insert(
MI.getOperand(0).getReg());
18433 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
18434 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
18435 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
18436 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
18437 Next->getOperand(5).isKill()) {
18442 SequenceMBBI != E; ++SequenceMBBI) {
18443 if (SequenceMBBI->isDebugInstr())
18446 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
18447 !SequenceMBBI->getOperand(2).isReg() ||
18448 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
18449 SequenceMBBI->getOperand(3).getImm() !=
CC ||
18450 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
18451 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
18453 LastSelectPseudo = &*SequenceMBBI;
18455 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
18458 if (SequenceMBBI->hasUnmodeledSideEffects() ||
18459 SequenceMBBI->mayLoadOrStore() ||
18460 SequenceMBBI->usesCustomInsertionHook())
18463 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
18478 F->insert(
I, IfFalseMBB);
18479 F->insert(
I, TailMBB);
18483 TailMBB->
push_back(DebugInstr->removeFromParent());
18487 TailMBB->
splice(TailMBB->
end(), HeadMBB,
18497 if (
MI.getOperand(2).isImm())
18500 .
addImm(
MI.getOperand(2).getImm())
18512 auto SelectMBBI =
MI.getIterator();
18513 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
18514 auto InsertionPoint = TailMBB->
begin();
18515 while (SelectMBBI != SelectEnd) {
18516 auto Next = std::next(SelectMBBI);
18519 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
18520 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
18521 .
addReg(SelectMBBI->getOperand(4).getReg())
18523 .
addReg(SelectMBBI->getOperand(5).getReg())
18538 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
18539 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
18541 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
18542 assert(
Masked &&
"Could not find masked instruction for LMUL and SEW pair");
18548 unsigned CVTXOpc) {
18554 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18567 .
add(
MI.getOperand(1))
18568 .
add(
MI.getOperand(2))
18569 .
add(
MI.getOperand(3))
18571 .
add(
MI.getOperand(4))
18572 .
add(
MI.getOperand(5))
18573 .
add(
MI.getOperand(6))
18588 .
add(
MI.getOperand(0))
18589 .
add(
MI.getOperand(1))
18591 .
add(
MI.getOperand(3))
18593 .
add(
MI.getOperand(4))
18594 .
add(
MI.getOperand(5))
18595 .
add(
MI.getOperand(6))
18605 MI.eraseFromParent();
18611 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
18613 switch (
MI.getOpcode()) {
18616 case RISCV::PseudoFROUND_H:
18617 CmpOpc = RISCV::FLT_H;
18618 F2IOpc = RISCV::FCVT_W_H;
18619 I2FOpc = RISCV::FCVT_H_W;
18620 FSGNJOpc = RISCV::FSGNJ_H;
18621 FSGNJXOpc = RISCV::FSGNJX_H;
18622 RC = &RISCV::FPR16RegClass;
18624 case RISCV::PseudoFROUND_H_INX:
18625 CmpOpc = RISCV::FLT_H_INX;
18626 F2IOpc = RISCV::FCVT_W_H_INX;
18627 I2FOpc = RISCV::FCVT_H_W_INX;
18628 FSGNJOpc = RISCV::FSGNJ_H_INX;
18629 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18630 RC = &RISCV::GPRF16RegClass;
18632 case RISCV::PseudoFROUND_S:
18633 CmpOpc = RISCV::FLT_S;
18634 F2IOpc = RISCV::FCVT_W_S;
18635 I2FOpc = RISCV::FCVT_S_W;
18636 FSGNJOpc = RISCV::FSGNJ_S;
18637 FSGNJXOpc = RISCV::FSGNJX_S;
18638 RC = &RISCV::FPR32RegClass;
18640 case RISCV::PseudoFROUND_S_INX:
18641 CmpOpc = RISCV::FLT_S_INX;
18642 F2IOpc = RISCV::FCVT_W_S_INX;
18643 I2FOpc = RISCV::FCVT_S_W_INX;
18644 FSGNJOpc = RISCV::FSGNJ_S_INX;
18645 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18646 RC = &RISCV::GPRF32RegClass;
18648 case RISCV::PseudoFROUND_D:
18650 CmpOpc = RISCV::FLT_D;
18651 F2IOpc = RISCV::FCVT_L_D;
18652 I2FOpc = RISCV::FCVT_D_L;
18653 FSGNJOpc = RISCV::FSGNJ_D;
18654 FSGNJXOpc = RISCV::FSGNJX_D;
18655 RC = &RISCV::FPR64RegClass;
18657 case RISCV::PseudoFROUND_D_INX:
18659 CmpOpc = RISCV::FLT_D_INX;
18660 F2IOpc = RISCV::FCVT_L_D_INX;
18661 I2FOpc = RISCV::FCVT_D_L_INX;
18662 FSGNJOpc = RISCV::FSGNJ_D_INX;
18663 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18664 RC = &RISCV::GPRRegClass;
18676 F->insert(
I, CvtMBB);
18677 F->insert(
I, DoneMBB);
18688 Register DstReg =
MI.getOperand(0).getReg();
18689 Register SrcReg =
MI.getOperand(1).getReg();
18690 Register MaxReg =
MI.getOperand(2).getReg();
18691 int64_t FRM =
MI.getOperand(3).getImm();
18696 Register FabsReg =
MRI.createVirtualRegister(RC);
18700 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18715 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18737 MI.eraseFromParent();
18744 switch (
MI.getOpcode()) {
18747 case RISCV::ReadCounterWide:
18749 "ReadCounterWide is only to be used on riscv32");
18751 case RISCV::Select_GPR_Using_CC_GPR:
18752 case RISCV::Select_GPR_Using_CC_Imm:
18753 case RISCV::Select_FPR16_Using_CC_GPR:
18754 case RISCV::Select_FPR16INX_Using_CC_GPR:
18755 case RISCV::Select_FPR32_Using_CC_GPR:
18756 case RISCV::Select_FPR32INX_Using_CC_GPR:
18757 case RISCV::Select_FPR64_Using_CC_GPR:
18758 case RISCV::Select_FPR64INX_Using_CC_GPR:
18759 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18761 case RISCV::BuildPairF64Pseudo:
18763 case RISCV::SplitF64Pseudo:
18765 case RISCV::PseudoQuietFLE_H:
18767 case RISCV::PseudoQuietFLE_H_INX:
18768 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18769 case RISCV::PseudoQuietFLT_H:
18771 case RISCV::PseudoQuietFLT_H_INX:
18772 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18773 case RISCV::PseudoQuietFLE_S:
18775 case RISCV::PseudoQuietFLE_S_INX:
18776 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18777 case RISCV::PseudoQuietFLT_S:
18779 case RISCV::PseudoQuietFLT_S_INX:
18780 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18781 case RISCV::PseudoQuietFLE_D:
18783 case RISCV::PseudoQuietFLE_D_INX:
18784 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18785 case RISCV::PseudoQuietFLE_D_IN32X:
18788 case RISCV::PseudoQuietFLT_D:
18790 case RISCV::PseudoQuietFLT_D_INX:
18791 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18792 case RISCV::PseudoQuietFLT_D_IN32X:
18796 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18798 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18800 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18802 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18804 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18806 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18808 case RISCV::PseudoFROUND_H:
18809 case RISCV::PseudoFROUND_H_INX:
18810 case RISCV::PseudoFROUND_S:
18811 case RISCV::PseudoFROUND_S_INX:
18812 case RISCV::PseudoFROUND_D:
18813 case RISCV::PseudoFROUND_D_INX:
18814 case RISCV::PseudoFROUND_D_IN32X:
18816 case TargetOpcode::STATEPOINT:
18822 MI.addOperand(*
MI.getMF(),
18828 case TargetOpcode::STACKMAP:
18829 case TargetOpcode::PATCHPOINT:
18832 "supported on 64-bit targets");
18850 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18882 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18883 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18886 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18887 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18890 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18891 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18895 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18896 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18897 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18899 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18900 RISCV::V20M2, RISCV::V22M2};
18908 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18909 RISCV::X13, RISCV::X14, RISCV::X15,
18910 RISCV::X16, RISCV::X17};
18912 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18913 RISCV::X13, RISCV::X14, RISCV::X15};
18925 static const MCPhysReg FastCCIGPRs[] = {
18926 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15,
18927 RISCV::X16, RISCV::X17, RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31};
18930 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18931 RISCV::X13, RISCV::X14, RISCV::X15};
18945 unsigned XLenInBytes = XLen / 8;
18958 Align StackAlign(XLenInBytes);
18959 if (!
EABI || XLen != 32)
18991 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18992 assert(XLen == 32 || XLen == 64);
18993 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18997 if (ArgFlags.
isNest()) {
18998 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
19006 if (!LocVT.
isVector() && IsRet && ValNo > 1)
19011 bool UseGPRForF16_F32 =
true;
19014 bool UseGPRForF64 =
true;
19026 UseGPRForF16_F32 = !IsFixed;
19030 UseGPRForF16_F32 = !IsFixed;
19031 UseGPRForF64 = !IsFixed;
19037 UseGPRForF16_F32 =
true;
19038 UseGPRForF64 =
true;
19045 if (UseGPRForF16_F32 &&
19046 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
19049 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
19066 unsigned TwoXLenInBytes = (2 * XLen) / 8;
19068 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
19072 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
19081 "PendingLocs and PendingArgFlags out of sync");
19085 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
19086 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
19134 PendingLocs.
size() <= 2) {
19135 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
19140 PendingLocs.
clear();
19141 PendingArgFlags.
clear();
19143 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
19149 unsigned StoreSizeBytes = XLen / 8;
19152 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
19154 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
19156 else if (ValVT == MVT::f64 && !UseGPRForF64)
19192 if (!PendingLocs.
empty()) {
19194 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
19196 for (
auto &It : PendingLocs) {
19198 It.convertToReg(Reg);
19203 PendingLocs.clear();
19204 PendingArgFlags.
clear();
19208 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
19210 "Expected an XLenVT or vector types at this stage");
19228template <
typename ArgTy>
19230 for (
const auto &ArgIdx :
enumerate(Args)) {
19231 MVT ArgVT = ArgIdx.value().VT;
19233 return ArgIdx.index();
19235 return std::nullopt;
19238void RISCVTargetLowering::analyzeInputArgs(
19241 RISCVCCAssignFn Fn)
const {
19242 unsigned NumArgs =
Ins.size();
19255 for (
unsigned i = 0; i != NumArgs; ++i) {
19259 Type *ArgTy =
nullptr;
19261 ArgTy = FType->getReturnType();
19262 else if (Ins[i].isOrigArg())
19263 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
19267 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
19269 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
19276void RISCVTargetLowering::analyzeOutputArgs(
19279 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
19280 unsigned NumArgs = Outs.
size();
19290 for (
unsigned i = 0; i != NumArgs; i++) {
19291 MVT ArgVT = Outs[i].VT;
19293 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
19297 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
19299 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
19353 if (In.isOrigArg()) {
19358 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
19359 (
BitWidth < 32 && In.Flags.isZExt())) {
19388 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
19434 ExtType,
DL, LocVT, Chain, FIN,
19451 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19464 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19474 unsigned ValNo,
MVT ValVT,
MVT LocVT,
19477 bool IsFixed,
bool IsRet,
Type *OrigTy,
19480 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19489 if (LocVT == MVT::f16 &&
19490 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
19492 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
19493 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
19494 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
19495 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
19496 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
19502 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19504 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
19505 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
19506 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
19507 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
19508 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19514 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19516 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
19517 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
19518 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
19519 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
19520 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19527 if ((LocVT == MVT::f16 &&
19528 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
19529 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19530 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
19531 Subtarget.hasStdExtZdinx())) {
19538 if (LocVT == MVT::f16) {
19544 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
19550 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
19558 if (AllocatedVReg) {
19593 if (ArgFlags.
isNest()) {
19595 "Attribute 'nest' is not supported in GHC calling convention");
19599 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
19600 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
19602 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19614 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19617 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
19618 RISCV::F18_F, RISCV::F19_F,
19619 RISCV::F20_F, RISCV::F21_F};
19620 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19626 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19629 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
19630 RISCV::F24_D, RISCV::F25_D,
19631 RISCV::F26_D, RISCV::F27_D};
19632 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19638 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19639 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
19659 switch (CallConv) {
19669 if (Subtarget.hasStdExtE())
19673 "(Zdinx/D) instruction set extensions");
19677 if (Func.hasFnAttribute(
"interrupt")) {
19678 if (!Func.arg_empty())
19680 "Functions with the interrupt attribute cannot have arguments!");
19685 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19687 "Function interrupt attribute argument not supported!");
19692 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19694 std::vector<SDValue> OutChains;
19703 analyzeInputArgs(MF, CCInfo, Ins,
false,
19707 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19728 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19729 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19731 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19733 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19762 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19767 if (VarArgsSaveSize == 0) {
19771 int VaArgOffset = -VarArgsSaveSize;
19779 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19780 VarArgsSaveSize += XLenInBytes;
19787 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19792 Chain,
DL, ArgValue, FIN,
19794 OutChains.push_back(Store);
19808 if (!OutChains.empty()) {
19809 OutChains.push_back(Chain);
19819bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19823 auto CalleeCC = CLI.CallConv;
19824 auto &Outs = CLI.Outs;
19826 auto CallerCC = Caller.getCallingConv();
19833 if (Caller.hasFnAttribute(
"interrupt"))
19848 for (
auto &VA : ArgLocs)
19854 auto IsCallerStructRet = Caller.hasStructRetAttr();
19855 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19856 if (IsCallerStructRet || IsCalleeStructRet)
19861 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19862 if (CalleeCC != CallerCC) {
19863 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19864 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19871 for (
auto &Arg : Outs)
19872 if (Arg.Flags.isByVal())
19907 if (Subtarget.hasStdExtE())
19911 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19917 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19923 "site marked musttail");
19930 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19932 if (!Flags.isByVal())
19936 unsigned Size = Flags.getByValSize();
19937 Align Alignment = Flags.getNonZeroByValAlign();
19944 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19946 false,
nullptr, IsTailCall,
19958 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19961 SDValue ArgValue = OutVals[OutIdx];
19981 if (!StackPtr.getNode())
19992 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
20010 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
20011 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
20017 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
20018 SDValue PartValue = OutVals[OutIdx + 1];
20019 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
20031 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
20033 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
20035 for (
const auto &Part : Parts) {
20036 SDValue PartValue = Part.first;
20037 SDValue PartOffset = Part.second;
20044 ArgValue = SpillSlot;
20050 if (Flags.isByVal())
20051 ArgValue = ByValArgs[j++];
20058 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
20059 "for passing parameters");
20062 if (!StackPtr.getNode())
20075 if (!MemOpChains.
empty())
20081 for (
auto &Reg : RegsToPass) {
20082 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
20089 validateCCReservedRegs(RegsToPass, MF);
20094 "Return address register required, but has been reserved."});
20113 for (
auto &Reg : RegsToPass)
20119 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
20120 assert(Mask &&
"Missing call preserved mask for calling convention");
20129 "Unexpected CFI type for a direct call");
20159 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
20160 auto &VA = RVLocs[i];
20168 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
20169 assert(VA.needsCustom());
20190 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
20194 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20195 MVT VT = Outs[i].VT;
20199 ArgFlags, CCInfo,
true,
true,
20200 nullptr, *
this, Dispatcher))
20232 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
20233 SDValue Val = OutVals[OutIdx];
20242 DAG.
getVTList(MVT::i32, MVT::i32), Val);
20246 Register RegHi = RVLocs[++i].getLocReg();
20252 "Return value register required, but has been reserved."});
20268 "Return value register required, but has been reserved."});
20290 if (Func.hasFnAttribute(
"interrupt")) {
20291 if (!Func.getReturnType()->isVoidTy())
20293 "Functions with the interrupt attribute must have void return type!");
20299 if (Kind ==
"supervisor")
20305 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
20308void RISCVTargetLowering::validateCCReservedRegs(
20309 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
20318 F,
"Argument register required, but has been reserved."});
20324 if (
N->getNumValues() != 1)
20326 if (!
N->hasNUsesOfValue(1, 0))
20329 SDNode *Copy = *
N->use_begin();
20343 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20347 bool HasRet =
false;
20348 for (
SDNode *Node : Copy->uses()) {
20356 Chain = Copy->getOperand(0);
20365#define NODE_NAME_CASE(NODE) \
20366 case RISCVISD::NODE: \
20367 return "RISCVISD::" #NODE;
20625#undef NODE_NAME_CASE
20632 if (Constraint.
size() == 1) {
20633 switch (Constraint[0]) {
20649 if (Constraint ==
"vr" || Constraint ==
"vm")
20655std::pair<unsigned, const TargetRegisterClass *>
20661 if (Constraint.
size() == 1) {
20662 switch (Constraint[0]) {
20667 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20668 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20669 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20670 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20671 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20672 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20673 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20675 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20676 return std::make_pair(0U, &RISCV::FPR16RegClass);
20677 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20678 return std::make_pair(0U, &RISCV::FPR32RegClass);
20679 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20680 return std::make_pair(0U, &RISCV::FPR64RegClass);
20685 }
else if (Constraint ==
"vr") {
20686 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
20687 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20689 return std::make_pair(0U, RC);
20691 }
else if (Constraint ==
"vm") {
20692 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20693 return std::make_pair(0U, &RISCV::VMV0RegClass);
20701 .
Case(
"{zero}", RISCV::X0)
20702 .
Case(
"{ra}", RISCV::X1)
20703 .
Case(
"{sp}", RISCV::X2)
20704 .
Case(
"{gp}", RISCV::X3)
20705 .
Case(
"{tp}", RISCV::X4)
20706 .
Case(
"{t0}", RISCV::X5)
20707 .
Case(
"{t1}", RISCV::X6)
20708 .
Case(
"{t2}", RISCV::X7)
20709 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20710 .
Case(
"{s1}", RISCV::X9)
20711 .
Case(
"{a0}", RISCV::X10)
20712 .
Case(
"{a1}", RISCV::X11)
20713 .
Case(
"{a2}", RISCV::X12)
20714 .
Case(
"{a3}", RISCV::X13)
20715 .
Case(
"{a4}", RISCV::X14)
20716 .
Case(
"{a5}", RISCV::X15)
20717 .
Case(
"{a6}", RISCV::X16)
20718 .
Case(
"{a7}", RISCV::X17)
20719 .
Case(
"{s2}", RISCV::X18)
20720 .
Case(
"{s3}", RISCV::X19)
20721 .
Case(
"{s4}", RISCV::X20)
20722 .
Case(
"{s5}", RISCV::X21)
20723 .
Case(
"{s6}", RISCV::X22)
20724 .
Case(
"{s7}", RISCV::X23)
20725 .
Case(
"{s8}", RISCV::X24)
20726 .
Case(
"{s9}", RISCV::X25)
20727 .
Case(
"{s10}", RISCV::X26)
20728 .
Case(
"{s11}", RISCV::X27)
20729 .
Case(
"{t3}", RISCV::X28)
20730 .
Case(
"{t4}", RISCV::X29)
20731 .
Case(
"{t5}", RISCV::X30)
20732 .
Case(
"{t6}", RISCV::X31)
20734 if (XRegFromAlias != RISCV::NoRegister)
20735 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20744 if (Subtarget.hasStdExtF()) {
20746 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20747 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20748 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20749 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20750 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20751 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20752 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20753 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20754 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20755 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20756 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20757 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20758 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20759 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20760 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20761 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20762 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20763 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20764 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20765 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20766 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20767 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20768 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20769 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20770 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20771 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20772 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20773 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20774 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20775 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20776 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20777 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20779 if (FReg != RISCV::NoRegister) {
20780 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20781 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20782 unsigned RegNo = FReg - RISCV::F0_F;
20783 unsigned DReg = RISCV::F0_D + RegNo;
20784 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20786 if (VT == MVT::f32 || VT == MVT::Other)
20787 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20788 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20789 unsigned RegNo = FReg - RISCV::F0_F;
20790 unsigned HReg = RISCV::F0_H + RegNo;
20791 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20798 .
Case(
"{v0}", RISCV::V0)
20799 .
Case(
"{v1}", RISCV::V1)
20800 .
Case(
"{v2}", RISCV::V2)
20801 .
Case(
"{v3}", RISCV::V3)
20802 .
Case(
"{v4}", RISCV::V4)
20803 .
Case(
"{v5}", RISCV::V5)
20804 .
Case(
"{v6}", RISCV::V6)
20805 .
Case(
"{v7}", RISCV::V7)
20806 .
Case(
"{v8}", RISCV::V8)
20807 .
Case(
"{v9}", RISCV::V9)
20808 .
Case(
"{v10}", RISCV::V10)
20809 .
Case(
"{v11}", RISCV::V11)
20810 .
Case(
"{v12}", RISCV::V12)
20811 .
Case(
"{v13}", RISCV::V13)
20812 .
Case(
"{v14}", RISCV::V14)
20813 .
Case(
"{v15}", RISCV::V15)
20814 .
Case(
"{v16}", RISCV::V16)
20815 .
Case(
"{v17}", RISCV::V17)
20816 .
Case(
"{v18}", RISCV::V18)
20817 .
Case(
"{v19}", RISCV::V19)
20818 .
Case(
"{v20}", RISCV::V20)
20819 .
Case(
"{v21}", RISCV::V21)
20820 .
Case(
"{v22}", RISCV::V22)
20821 .
Case(
"{v23}", RISCV::V23)
20822 .
Case(
"{v24}", RISCV::V24)
20823 .
Case(
"{v25}", RISCV::V25)
20824 .
Case(
"{v26}", RISCV::V26)
20825 .
Case(
"{v27}", RISCV::V27)
20826 .
Case(
"{v28}", RISCV::V28)
20827 .
Case(
"{v29}", RISCV::V29)
20828 .
Case(
"{v30}", RISCV::V30)
20829 .
Case(
"{v31}", RISCV::V31)
20831 if (VReg != RISCV::NoRegister) {
20832 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20833 return std::make_pair(VReg, &RISCV::VMRegClass);
20834 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20835 return std::make_pair(VReg, &RISCV::VRRegClass);
20836 for (
const auto *RC :
20837 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20838 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20839 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20840 return std::make_pair(VReg, RC);
20846 std::pair<Register, const TargetRegisterClass *> Res =
20852 if (Res.second == &RISCV::GPRF16RegClass ||
20853 Res.second == &RISCV::GPRF32RegClass ||
20854 Res.second == &RISCV::GPRPairRegClass)
20855 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20863 if (ConstraintCode.
size() == 1) {
20864 switch (ConstraintCode[0]) {
20879 if (Constraint.
size() == 1) {
20880 switch (Constraint[0]) {
20883 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20885 if (isInt<12>(CVal))
20898 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20900 if (isUInt<5>(CVal))
20918 if (Subtarget.hasStdExtZtso()) {
20934 if (Subtarget.hasStdExtZtso()) {
20942 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Inst) &&
20959 if (Subtarget.hasForcedAtomics())
20964 if (Subtarget.hasStdExtZacas() &&
20965 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20971 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20984 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20986 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20988 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20990 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20992 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20994 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20996 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20998 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
21007 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
21009 return Intrinsic::riscv_masked_atomicrmw_add_i64;
21011 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
21013 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
21015 return Intrinsic::riscv_masked_atomicrmw_max_i64;
21017 return Intrinsic::riscv_masked_atomicrmw_min_i64;
21019 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
21021 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
21047 unsigned XLen = Subtarget.
getXLen();
21071 unsigned ValWidth =
21076 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
21079 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
21091 if (Subtarget.hasForcedAtomics())
21095 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
21104 unsigned XLen = Subtarget.
getXLen();
21106 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
21111 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
21117 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
21124 EVT DataVT)
const {
21140 return Subtarget.hasStdExtZfhmin();
21142 return Subtarget.hasStdExtF();
21144 return Subtarget.hasStdExtD();
21176 "RVVBitsPerBlock changed, audit needed");
21185 if (!Subtarget.hasVendorXTHeadMemIdx())
21191 Base =
Op->getOperand(0);
21193 int64_t RHSC =
RHS->getSExtValue();
21199 bool isLegalIndexedOffset =
false;
21200 for (
unsigned i = 0; i < 4; i++)
21201 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
21202 isLegalIndexedOffset =
true;
21206 if (!isLegalIndexedOffset)
21223 VT = LD->getMemoryVT();
21224 Ptr = LD->getBasePtr();
21225 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21226 VT = ST->getMemoryVT();
21227 Ptr = ST->getBasePtr();
21243 if (Subtarget.hasVendorXCVmem()) {
21248 Base = LS->getBasePtr();
21252 if (
Base ==
Op->getOperand(0))
21254 else if (
Base ==
Op->getOperand(1))
21266 VT = LD->getMemoryVT();
21267 Ptr = LD->getBasePtr();
21268 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21269 VT = ST->getMemoryVT();
21270 Ptr = ST->getBasePtr();
21313 const Constant *PersonalityFn)
const {
21318 const Constant *PersonalityFn)
const {
21342 const bool HasZmmul = Subtarget.hasStdExtZmmul();
21351 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
21353 const APInt &Imm = ConstNode->getAPIntValue();
21354 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
21355 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
21359 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
21360 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
21361 (Imm - 8).isPowerOf2()))
21366 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
21367 ConstNode->hasOneUse()) {
21368 APInt ImmS = Imm.ashr(Imm.countr_zero());
21369 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
21370 (1 - ImmS).isPowerOf2())
21394 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
21403 unsigned *
Fast)
const {
21406 *
Fast = Subtarget.enableUnalignedScalarMem();
21407 return Subtarget.enableUnalignedScalarMem();
21423 *
Fast = Subtarget.enableUnalignedVectorMem();
21424 return Subtarget.enableUnalignedVectorMem();
21433 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
21445 if (
Op.size() < MinVLenInBytes)
21455 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
21459 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
21461 if (
Op.isFixedDstAlign())
21462 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
21464 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
21472 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
21473 bool IsABIRegCopy =
CC.has_value();
21475 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21476 PartVT == MVT::f32) {
21494 if (PartVTBitSize % ValueVTBitSize == 0) {
21495 assert(PartVTBitSize >= ValueVTBitSize);
21502 if (ValueEltVT != PartEltVT) {
21503 if (PartVTBitSize > ValueVTBitSize) {
21505 assert(Count != 0 &&
"The number of element should not be zero.");
21506 EVT SameEltTypeVT =
21527 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
21528 bool IsABIRegCopy =
CC.has_value();
21529 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21530 PartVT == MVT::f32) {
21547 if (PartVTBitSize % ValueVTBitSize == 0) {
21548 assert(PartVTBitSize >= ValueVTBitSize);
21549 EVT SameEltTypeVT = ValueVT;
21556 if (ValueEltVT != PartEltVT) {
21558 assert(Count != 0 &&
"The number of element should not be zero.");
21575 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21582 unsigned Opc =
N->getOpcode();
21612 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21626 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21631 if (FVTy->getNumElements() < 2)
21646 return Factor * LMUL <= 8;
21650 Align Alignment)
const {
21662 if (!Subtarget.enableUnalignedVectorMem() &&
21670 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21671 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21672 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21673 Intrinsic::riscv_seg8_load};
21692 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21702 {VTy, LI->getPointerOperandType(), XLenTy});
21704 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21709 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21711 Shuffles[i]->replaceAllUsesWith(SubVec);
21718 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21719 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21720 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21721 Intrinsic::riscv_seg8_store};
21741 unsigned Factor)
const {
21743 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21746 ShuffleVTy->getNumElements() / Factor);
21748 SI->getPointerAddressSpace(),
21749 SI->getDataLayout()))
21756 {VTy, SI->getPointerOperandType(), XLenTy});
21761 for (
unsigned i = 0; i < Factor; i++) {
21770 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21771 Ops.
append({SI->getPointerOperand(), VL});
21787 unsigned Factor = 2;
21802 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21805 {ResVTy, LI->getPointerOperandType(), XLenTy});
21806 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21809 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21810 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21811 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21812 Intrinsic::riscv_vlseg8};
21834 if (
II->getIntrinsicID() != Intrinsic::vector_interleave2)
21837 unsigned Factor = 2;
21840 VectorType *InVTy = cast<VectorType>(
II->getOperand(0)->getType());
21843 SI->getPointerAddressSpace(),
21844 SI->getDataLayout()))
21851 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21854 {InVTy, SI->getPointerOperandType(), XLenTy});
21855 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21858 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21859 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21860 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21861 Intrinsic::riscv_vsseg8};
21868 Builder.
CreateCall(VssegNFunc, {
II->getOperand(0),
II->getOperand(1),
21869 SI->getPointerOperand(), VL});
21879 "Invalid call instruction for a KCFI check");
21881 MBBI->getOpcode()));
21884 Target.setIsRenamable(
false);
21892#define GET_REGISTER_MATCHER
21893#include "RISCVGenAsmMatcher.inc"
21899 if (Reg == RISCV::NoRegister)
21901 if (Reg == RISCV::NoRegister)
21913 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21915 if (NontemporalInfo ==
nullptr)
21923 int NontemporalLevel = 5;
21924 const MDNode *RISCVNontemporalInfo =
21925 I.getMetadata(
"riscv-nontemporal-domain");
21926 if (RISCVNontemporalInfo !=
nullptr)
21929 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21933 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21934 "RISC-V target doesn't support this non-temporal domain.");
21936 NontemporalLevel -= 2;
21938 if (NontemporalLevel & 0b1)
21940 if (NontemporalLevel & 0b10)
21953 return TargetFlags;
21963 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21966 return Subtarget.hasStdExtZbb() &&
21979 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21980 Op == Instruction::And ||
Op == Instruction::Or ||
21981 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21982 Op == Instruction::ShuffleVector ||
Op == Instruction::Load ||
21983 Op == Instruction::Freeze ||
Op == Instruction::Store)
21991 !isa<ReturnInst>(&Inst))
21994 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21995 if (AI->getAllocatedType()->isScalableTy())
22003RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
22011 if (!Subtarget.hasShortForwardBranchOpt())
22013 EVT VT =
N->getValueType(0);
22014 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
22018 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
22023bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
22024 EVT VT,
const APInt &AndMask)
const {
22025 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
22026 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
22030unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
22035template <
typename Arg>
22036void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
22039 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
22041 auto It = ArgList.
begin();
22042 MVT FirstArgRegType = It->VT;
22045 if (It == ArgList.
end() || It->Flags.isSplit())
22057 for (; It != ArgList.
end(); ++It)
22058 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
22064 if (isHomogeneousScalableVectorType(ArgList)) {
22066 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
22069 bool FirstVMaskAssigned =
false;
22070 for (
const auto &OutArg : ArgList) {
22071 MVT RegisterVT = OutArg.VT;
22081 RVVArgInfos.push_back({1, RegisterVT,
true});
22082 FirstVMaskAssigned =
true;
22086 RVVArgInfos.push_back({1, RegisterVT,
false});
22093void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
22098 bool FirstVMaskAssigned =
false;
22099 for (
Type *Ty : TypeList) {
22109 RVVArgInfos.push_back(
22115 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
22130 if (!FirstVMaskAssigned &&
22132 RVVArgInfos.push_back({1, RegisterVT,
true});
22133 FirstVMaskAssigned =
true;
22137 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
22143void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
22144 unsigned StartReg) {
22145 assert((StartReg % LMul) == 0 &&
22146 "Start register number should be multiple of lmul");
22165 for (
unsigned i = 0; i < NF; ++i)
22167 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
22169 AllocatedPhysRegs.push_back(
MCPhysReg());
22175void RVVArgDispatcher::compute() {
22177 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
22180 AllocatedPhysRegs.push_back(RISCV::V0);
22186 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
22187 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
22188 StartReg += RegsNeeded) {
22189 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
22190 if ((AssignedMap & Map) == 0) {
22191 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
22192 AssignedMap |=
Map;
22197 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
22200 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
22201 allocate(RVVArgInfos[i]);
22205 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
22206 return AllocatedPhysRegs[CurIdx++];
22213 if (Subtarget.hasStdExtZicfilp()) {
22225#define GET_RISCVVIntrinsicsTable_IMPL
22226#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static unsigned getPACKOpcode(unsigned DestBW, const RISCVSubtarget &Subtarget)
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static SDValue lowerBUILD_VECTORvXf16(SDValue Op, SelectionDAG &DAG)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Double the element size of the build vector to reduce the number of vslide1down in the build vector c...
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool isTargetAndroid() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SW_GUARDED_BRIND
Software guarded BRIND node.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost, bool FreeZeroes)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)