37#include "llvm/IR/IntrinsicsRISCV.h"
52#define DEBUG_TYPE "riscv-lower"
58 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
59 "instructions that we will consider for VW expansion"),
64 cl::desc(
"Allow the formation of VW_W operations (e.g., "
65 "VWADD_W) with splat constants"),
70 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
71 "transformation to multiplications by the reciprocal"),
76 cl::desc(
"Give the maximum number of instructions that we will "
77 "use for creating a floating-point immediate value"),
88 !Subtarget.hasStdExtF()) {
89 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
90 "doesn't support the F instruction set extension (ignoring "
94 !Subtarget.hasStdExtD()) {
95 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
96 "doesn't support the D instruction set extension (ignoring "
120 if (Subtarget.hasStdExtZfhmin())
122 if (Subtarget.hasStdExtZfbfmin())
124 if (Subtarget.hasStdExtF())
126 if (Subtarget.hasStdExtD())
128 if (Subtarget.hasStdExtZhinxmin())
130 if (Subtarget.hasStdExtZfinx())
132 if (Subtarget.hasStdExtZdinx()) {
140 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
141 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
143 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
144 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
145 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
146 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
147 MVT::nxv4i64, MVT::nxv8i64};
149 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
150 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
152 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
153 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
155 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
157 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
159 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
160 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
161 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
162 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
163 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
164 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
165 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
166 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
167 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
168 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
169 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
172 auto addRegClassForRVV = [
this](
MVT VT) {
176 if (VT.getVectorMinNumElements() < MinElts)
179 unsigned Size = VT.getSizeInBits().getKnownMinValue();
182 RC = &RISCV::VRRegClass;
184 RC = &RISCV::VRM2RegClass;
186 RC = &RISCV::VRM4RegClass;
188 RC = &RISCV::VRM8RegClass;
195 for (
MVT VT : BoolVecVTs)
196 addRegClassForRVV(VT);
197 for (
MVT VT : IntVecVTs) {
198 if (VT.getVectorElementType() == MVT::i64 &&
201 addRegClassForRVV(VT);
205 for (
MVT VT : F16VecVTs)
206 addRegClassForRVV(VT);
209 for (
MVT VT : BF16VecVTs)
210 addRegClassForRVV(VT);
213 for (
MVT VT : F32VecVTs)
214 addRegClassForRVV(VT);
217 for (
MVT VT : F64VecVTs)
218 addRegClassForRVV(VT);
221 auto addRegClassForFixedVectors = [
this](
MVT VT) {
228 if (useRVVForFixedLengthVectorVT(VT))
229 addRegClassForFixedVectors(VT);
232 if (useRVVForFixedLengthVectorVT(VT))
233 addRegClassForFixedVectors(VT);
293 if (!(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
303 if (!Subtarget.hasVendorXTHeadBb())
308 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
309 !(Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()))
319 if (!Subtarget.hasStdExtZbb())
325 if (!Subtarget.hasStdExtZmmul()) {
327 }
else if (Subtarget.
is64Bit()) {
334 if (!Subtarget.hasStdExtM()) {
337 }
else if (Subtarget.
is64Bit()) {
339 {MVT::i8, MVT::i16, MVT::i32},
Custom);
349 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
352 }
else if (Subtarget.hasVendorXTHeadBb()) {
356 }
else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
365 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
366 Subtarget.hasVendorXTHeadBb())
370 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit()) {
378 if (Subtarget.hasStdExtZbb() ||
379 (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit())) {
384 if (Subtarget.hasStdExtZbb() ||
385 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
392 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
393 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit())) {
402 if (Subtarget.hasVendorXCValu() && !Subtarget.
is64Bit()) {
404 }
else if (Subtarget.hasShortForwardBranchOpt()) {
407 }
else if (Subtarget.
is64Bit()) {
411 if (!Subtarget.hasVendorXTHeadCondMov())
414 static const unsigned FPLegalNodeTypes[] = {
428 static const unsigned FPOpToExpand[] = {
432 static const unsigned FPRndMode[] = {
436 static const unsigned ZfhminZfbfminPromoteOps[] = {
447 if (Subtarget.hasStdExtZfbfmin()) {
470 if (Subtarget.hasStdExtZfa())
497 Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ?
Legal :
Promote);
539 if (Subtarget.hasStdExtZfa()) {
557 if (Subtarget.hasStdExtZfa()) {
639 if (Subtarget.hasStdExtZicbop()) {
643 if (Subtarget.hasStdExtA()) {
645 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
649 }
else if (Subtarget.hasForcedAtomics()) {
673 {MVT::i8, MVT::i16},
Custom);
684 static const unsigned IntegerVPOps[] = {
685 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
686 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
687 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
688 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
689 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
690 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
691 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
692 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
693 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
694 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
695 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
696 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
697 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
698 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
699 ISD::EXPERIMENTAL_VP_SPLAT};
701 static const unsigned FloatingPointVPOps[] = {
702 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
703 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
704 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
705 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
706 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
707 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
708 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
709 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
710 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
711 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
712 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
713 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
714 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
715 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
717 static const unsigned IntegerVecReduceOps[] = {
722 static const unsigned FloatingPointVecReduceOps[] = {
726 static const unsigned FloatingPointLibCallOps[] = {
739 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
740 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
741 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
745 for (
MVT VT : BoolVecVTs) {
775 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
799 ISD::VP_TRUNCATE, ISD::VP_SETCC},
815 for (
MVT VT : IntVecVTs) {
826 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
876 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
877 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
901 if (Subtarget.hasStdExtZvkb()) {
909 if (Subtarget.hasStdExtZvbb()) {
913 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
919 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
928 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
936 for (
MVT VT : VecTupleVTs) {
957 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
967 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
982 ISD::VP_FROUNDTOZERO,
988 ISD::VP_REDUCE_FMINIMUM,
989 ISD::VP_REDUCE_FMAXIMUM};
992 const auto SetCommonVFPActions = [&](
MVT VT) {
1027 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1028 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1061 const auto SetCommonVFPExtLoadTruncStoreActions =
1063 for (
auto SmallVT : SmallerVTs) {
1071 const auto SetCommonPromoteToF32Actions = [&](
MVT VT) {
1096 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1097 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1120 for (
MVT VT : F16VecVTs) {
1123 SetCommonVFPActions(VT);
1126 for (
MVT VT : F16VecVTs) {
1129 SetCommonPromoteToF32Actions(VT);
1134 for (
MVT VT : BF16VecVTs) {
1137 SetCommonPromoteToF32Actions(VT);
1142 for (
MVT VT : F32VecVTs) {
1145 SetCommonVFPActions(VT);
1146 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1147 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1152 for (
MVT VT : F64VecVTs) {
1155 SetCommonVFPActions(VT);
1156 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1157 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1158 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1164 if (!useRVVForFixedLengthVectorVT(VT))
1210 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1237 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1261 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1262 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1299 if (Subtarget.hasStdExtZvkb())
1302 if (Subtarget.hasStdExtZvbb()) {
1326 if (!useRVVForFixedLengthVectorVT(VT))
1347 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1348 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1364 if (Subtarget.hasStdExtZfhmin()) {
1388 if (Subtarget.hasStdExtZfbfmin()) {
1448 if (Subtarget.hasStdExtZfbfmin())
1457 if (Subtarget.hasStdExtA())
1460 if (Subtarget.hasForcedAtomics()) {
1470 if (Subtarget.hasVendorXTHeadMemIdx()) {
1486 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
1512 if (Subtarget.hasStdExtZbb())
1515 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1519 if (Subtarget.hasStdExtZbkb())
1530 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1533 if (Subtarget.hasVendorXTHeadMemPair())
1576MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1581bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1583 bool IsScalable)
const {
1590 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1615 unsigned Intrinsic)
const {
1616 auto &
DL =
I.getDataLayout();
1618 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1619 bool IsUnitStrided,
bool UsePtrVal =
false) {
1624 Info.ptrVal =
I.getArgOperand(PtrOp);
1626 Info.fallbackAddressSpace =
1627 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1631 MemTy =
I.getArgOperand(0)->getType();
1634 MemTy =
I.getType();
1644 if (cast<TargetExtType>(MemTy)->
getName() ==
"riscv.vector.tuple")
1647 1 << cast<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))
1649 Info.align =
DL.getABITypeAlign(MemTy);
1659 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1663 switch (Intrinsic) {
1666 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1667 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1668 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1669 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1670 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1671 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1672 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1673 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1674 case Intrinsic::riscv_masked_cmpxchg_i32:
1676 Info.memVT = MVT::i32;
1677 Info.ptrVal =
I.getArgOperand(0);
1683 case Intrinsic::riscv_seg2_load:
1684 case Intrinsic::riscv_seg3_load:
1685 case Intrinsic::riscv_seg4_load:
1686 case Intrinsic::riscv_seg5_load:
1687 case Intrinsic::riscv_seg6_load:
1688 case Intrinsic::riscv_seg7_load:
1689 case Intrinsic::riscv_seg8_load:
1690 return SetRVVLoadStoreInfo( 0,
false,
1692 case Intrinsic::riscv_seg2_store:
1693 case Intrinsic::riscv_seg3_store:
1694 case Intrinsic::riscv_seg4_store:
1695 case Intrinsic::riscv_seg5_store:
1696 case Intrinsic::riscv_seg6_store:
1697 case Intrinsic::riscv_seg7_store:
1698 case Intrinsic::riscv_seg8_store:
1700 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1703 case Intrinsic::riscv_vle:
1704 case Intrinsic::riscv_vle_mask:
1705 case Intrinsic::riscv_vleff:
1706 case Intrinsic::riscv_vleff_mask:
1707 return SetRVVLoadStoreInfo( 1,
1711 case Intrinsic::riscv_vse:
1712 case Intrinsic::riscv_vse_mask:
1713 return SetRVVLoadStoreInfo( 1,
1717 case Intrinsic::riscv_vlse:
1718 case Intrinsic::riscv_vlse_mask:
1719 case Intrinsic::riscv_vloxei:
1720 case Intrinsic::riscv_vloxei_mask:
1721 case Intrinsic::riscv_vluxei:
1722 case Intrinsic::riscv_vluxei_mask:
1723 return SetRVVLoadStoreInfo( 1,
1726 case Intrinsic::riscv_vsse:
1727 case Intrinsic::riscv_vsse_mask:
1728 case Intrinsic::riscv_vsoxei:
1729 case Intrinsic::riscv_vsoxei_mask:
1730 case Intrinsic::riscv_vsuxei:
1731 case Intrinsic::riscv_vsuxei_mask:
1732 return SetRVVLoadStoreInfo( 1,
1735 case Intrinsic::riscv_vlseg2:
1736 case Intrinsic::riscv_vlseg3:
1737 case Intrinsic::riscv_vlseg4:
1738 case Intrinsic::riscv_vlseg5:
1739 case Intrinsic::riscv_vlseg6:
1740 case Intrinsic::riscv_vlseg7:
1741 case Intrinsic::riscv_vlseg8:
1742 case Intrinsic::riscv_vlseg2ff:
1743 case Intrinsic::riscv_vlseg3ff:
1744 case Intrinsic::riscv_vlseg4ff:
1745 case Intrinsic::riscv_vlseg5ff:
1746 case Intrinsic::riscv_vlseg6ff:
1747 case Intrinsic::riscv_vlseg7ff:
1748 case Intrinsic::riscv_vlseg8ff:
1749 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1752 case Intrinsic::riscv_vlseg2_mask:
1753 case Intrinsic::riscv_vlseg3_mask:
1754 case Intrinsic::riscv_vlseg4_mask:
1755 case Intrinsic::riscv_vlseg5_mask:
1756 case Intrinsic::riscv_vlseg6_mask:
1757 case Intrinsic::riscv_vlseg7_mask:
1758 case Intrinsic::riscv_vlseg8_mask:
1759 case Intrinsic::riscv_vlseg2ff_mask:
1760 case Intrinsic::riscv_vlseg3ff_mask:
1761 case Intrinsic::riscv_vlseg4ff_mask:
1762 case Intrinsic::riscv_vlseg5ff_mask:
1763 case Intrinsic::riscv_vlseg6ff_mask:
1764 case Intrinsic::riscv_vlseg7ff_mask:
1765 case Intrinsic::riscv_vlseg8ff_mask:
1766 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1769 case Intrinsic::riscv_vlsseg2:
1770 case Intrinsic::riscv_vlsseg3:
1771 case Intrinsic::riscv_vlsseg4:
1772 case Intrinsic::riscv_vlsseg5:
1773 case Intrinsic::riscv_vlsseg6:
1774 case Intrinsic::riscv_vlsseg7:
1775 case Intrinsic::riscv_vlsseg8:
1776 case Intrinsic::riscv_vloxseg2:
1777 case Intrinsic::riscv_vloxseg3:
1778 case Intrinsic::riscv_vloxseg4:
1779 case Intrinsic::riscv_vloxseg5:
1780 case Intrinsic::riscv_vloxseg6:
1781 case Intrinsic::riscv_vloxseg7:
1782 case Intrinsic::riscv_vloxseg8:
1783 case Intrinsic::riscv_vluxseg2:
1784 case Intrinsic::riscv_vluxseg3:
1785 case Intrinsic::riscv_vluxseg4:
1786 case Intrinsic::riscv_vluxseg5:
1787 case Intrinsic::riscv_vluxseg6:
1788 case Intrinsic::riscv_vluxseg7:
1789 case Intrinsic::riscv_vluxseg8:
1790 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1793 case Intrinsic::riscv_vlsseg2_mask:
1794 case Intrinsic::riscv_vlsseg3_mask:
1795 case Intrinsic::riscv_vlsseg4_mask:
1796 case Intrinsic::riscv_vlsseg5_mask:
1797 case Intrinsic::riscv_vlsseg6_mask:
1798 case Intrinsic::riscv_vlsseg7_mask:
1799 case Intrinsic::riscv_vlsseg8_mask:
1800 case Intrinsic::riscv_vloxseg2_mask:
1801 case Intrinsic::riscv_vloxseg3_mask:
1802 case Intrinsic::riscv_vloxseg4_mask:
1803 case Intrinsic::riscv_vloxseg5_mask:
1804 case Intrinsic::riscv_vloxseg6_mask:
1805 case Intrinsic::riscv_vloxseg7_mask:
1806 case Intrinsic::riscv_vloxseg8_mask:
1807 case Intrinsic::riscv_vluxseg2_mask:
1808 case Intrinsic::riscv_vluxseg3_mask:
1809 case Intrinsic::riscv_vluxseg4_mask:
1810 case Intrinsic::riscv_vluxseg5_mask:
1811 case Intrinsic::riscv_vluxseg6_mask:
1812 case Intrinsic::riscv_vluxseg7_mask:
1813 case Intrinsic::riscv_vluxseg8_mask:
1814 return SetRVVLoadStoreInfo(
I.arg_size() - 6,
1817 case Intrinsic::riscv_vsseg2:
1818 case Intrinsic::riscv_vsseg3:
1819 case Intrinsic::riscv_vsseg4:
1820 case Intrinsic::riscv_vsseg5:
1821 case Intrinsic::riscv_vsseg6:
1822 case Intrinsic::riscv_vsseg7:
1823 case Intrinsic::riscv_vsseg8:
1824 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1827 case Intrinsic::riscv_vsseg2_mask:
1828 case Intrinsic::riscv_vsseg3_mask:
1829 case Intrinsic::riscv_vsseg4_mask:
1830 case Intrinsic::riscv_vsseg5_mask:
1831 case Intrinsic::riscv_vsseg6_mask:
1832 case Intrinsic::riscv_vsseg7_mask:
1833 case Intrinsic::riscv_vsseg8_mask:
1834 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1837 case Intrinsic::riscv_vssseg2:
1838 case Intrinsic::riscv_vssseg3:
1839 case Intrinsic::riscv_vssseg4:
1840 case Intrinsic::riscv_vssseg5:
1841 case Intrinsic::riscv_vssseg6:
1842 case Intrinsic::riscv_vssseg7:
1843 case Intrinsic::riscv_vssseg8:
1844 case Intrinsic::riscv_vsoxseg2:
1845 case Intrinsic::riscv_vsoxseg3:
1846 case Intrinsic::riscv_vsoxseg4:
1847 case Intrinsic::riscv_vsoxseg5:
1848 case Intrinsic::riscv_vsoxseg6:
1849 case Intrinsic::riscv_vsoxseg7:
1850 case Intrinsic::riscv_vsoxseg8:
1851 case Intrinsic::riscv_vsuxseg2:
1852 case Intrinsic::riscv_vsuxseg3:
1853 case Intrinsic::riscv_vsuxseg4:
1854 case Intrinsic::riscv_vsuxseg5:
1855 case Intrinsic::riscv_vsuxseg6:
1856 case Intrinsic::riscv_vsuxseg7:
1857 case Intrinsic::riscv_vsuxseg8:
1858 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1861 case Intrinsic::riscv_vssseg2_mask:
1862 case Intrinsic::riscv_vssseg3_mask:
1863 case Intrinsic::riscv_vssseg4_mask:
1864 case Intrinsic::riscv_vssseg5_mask:
1865 case Intrinsic::riscv_vssseg6_mask:
1866 case Intrinsic::riscv_vssseg7_mask:
1867 case Intrinsic::riscv_vssseg8_mask:
1868 case Intrinsic::riscv_vsoxseg2_mask:
1869 case Intrinsic::riscv_vsoxseg3_mask:
1870 case Intrinsic::riscv_vsoxseg4_mask:
1871 case Intrinsic::riscv_vsoxseg5_mask:
1872 case Intrinsic::riscv_vsoxseg6_mask:
1873 case Intrinsic::riscv_vsoxseg7_mask:
1874 case Intrinsic::riscv_vsoxseg8_mask:
1875 case Intrinsic::riscv_vsuxseg2_mask:
1876 case Intrinsic::riscv_vsuxseg3_mask:
1877 case Intrinsic::riscv_vsuxseg4_mask:
1878 case Intrinsic::riscv_vsuxseg5_mask:
1879 case Intrinsic::riscv_vsuxseg6_mask:
1880 case Intrinsic::riscv_vsuxseg7_mask:
1881 case Intrinsic::riscv_vsuxseg8_mask:
1882 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1923 return isInt<12>(Imm);
1927 return isInt<12>(Imm);
1940 return (SrcBits == 64 && DestBits == 32);
1951 return (SrcBits == 64 && DestBits == 32);
1962 if (SrcBits == DestBits * 2) {
1973 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1974 EVT MemVT = LD->getMemoryVT();
1975 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1985 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1993 return Subtarget.hasStdExtZbb() ||
1994 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
1998 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1999 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.
is64Bit());
2010 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
2015 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
2019 EVT VT =
Y.getValueType();
2025 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2026 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
2031 if (Subtarget.hasStdExtZbs())
2032 return X.getValueType().isScalarInteger();
2033 auto *
C = dyn_cast<ConstantSDNode>(
Y);
2035 if (Subtarget.hasVendorXTHeadBs())
2036 return C !=
nullptr;
2038 return C &&
C->getAPIntValue().ule(10);
2058 if (BitSize > Subtarget.
getXLen())
2062 int64_t Val = Imm.getSExtValue();
2070 if (!Subtarget.enableUnalignedScalarMem())
2086 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
2093 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
2097 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2141 if (!Subtarget.hasStdExtZfa())
2144 bool IsSupportedVT =
false;
2145 if (VT == MVT::f16) {
2146 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2147 }
else if (VT == MVT::f32) {
2148 IsSupportedVT =
true;
2149 }
else if (VT == MVT::f64) {
2150 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2151 IsSupportedVT =
true;
2161 bool ForCodeSize)
const {
2162 bool IsLegalVT =
false;
2165 else if (VT == MVT::f32)
2167 else if (VT == MVT::f64)
2169 else if (VT == MVT::bf16)
2170 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2182 return Imm.isZero();
2186 if (Imm.isNegZero())
2191 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2194 Subtarget.
getXLen(), Subtarget);
2200 unsigned Index)
const {
2213 if (EltVT == MVT::i1)
2226 if (Index + ResElts <= MinVLMAX && Index < 31)
2235 return (ResElts * 2) == SrcElts && (Index == 0 || Index == ResElts);
2254 std::optional<MVT> RegisterVT)
const {
2256 if (VT == (Subtarget.
is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2257 *RegisterVT == MVT::Untyped)
2277 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2279 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2294 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2300 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2313 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2314 int64_t
C = RHSC->getSExtValue();
2352 if (VT.
SimpleTy >= MVT::riscv_nxv1i8x2 &&
2353 VT.
SimpleTy <= MVT::riscv_nxv1i8x8)
2355 if (VT.
SimpleTy >= MVT::riscv_nxv2i8x2 &&
2356 VT.
SimpleTy <= MVT::riscv_nxv2i8x8)
2358 if (VT.
SimpleTy >= MVT::riscv_nxv4i8x2 &&
2359 VT.
SimpleTy <= MVT::riscv_nxv4i8x8)
2361 if (VT.
SimpleTy >= MVT::riscv_nxv8i8x2 &&
2362 VT.
SimpleTy <= MVT::riscv_nxv8i8x8)
2364 if (VT.
SimpleTy >= MVT::riscv_nxv16i8x2 &&
2365 VT.
SimpleTy <= MVT::riscv_nxv16i8x4)
2367 if (VT.
SimpleTy == MVT::riscv_nxv32i8x2)
2377 switch (KnownSize) {
2405 return RISCV::VRRegClassID;
2407 return RISCV::VRM2RegClassID;
2409 return RISCV::VRM4RegClassID;
2411 return RISCV::VRM8RegClassID;
2421 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2422 "Unexpected subreg numbering");
2423 return RISCV::sub_vrm1_0 + Index;
2426 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2427 "Unexpected subreg numbering");
2428 return RISCV::sub_vrm2_0 + Index;
2431 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2432 "Unexpected subreg numbering");
2433 return RISCV::sub_vrm4_0 + Index;
2441 unsigned RegsPerField =
2444 switch (RegsPerField) {
2447 return RISCV::VRN2M1RegClassID;
2449 return RISCV::VRN3M1RegClassID;
2451 return RISCV::VRN4M1RegClassID;
2453 return RISCV::VRN5M1RegClassID;
2455 return RISCV::VRN6M1RegClassID;
2457 return RISCV::VRN7M1RegClassID;
2459 return RISCV::VRN8M1RegClassID;
2463 return RISCV::VRN2M2RegClassID;
2465 return RISCV::VRN3M2RegClassID;
2467 return RISCV::VRN4M2RegClassID;
2471 return RISCV::VRN2M4RegClassID;
2479 return RISCV::VRRegClassID;
2488std::pair<unsigned, unsigned>
2490 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2492 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2493 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2494 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2495 "Register classes not ordered");
2502 if (VecRegClassID == SubRegClassID)
2503 return {RISCV::NoSubRegister, 0};
2506 "Only allow scalable vector subvector.");
2508 "Invalid vector tuple insert/extract for vector and subvector with "
2519 unsigned SubRegIdx = RISCV::NoSubRegister;
2520 for (
const unsigned RCID :
2521 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2522 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2526 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2531 return {SubRegIdx, InsertExtractIdx};
2536bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2567unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2574 "Unexpected opcode");
2576 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2578 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2581 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2655bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2656 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2665 "Expected legal fixed length vector!");
2668 unsigned MaxELen = Subtarget.
getELen();
2702 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2709 "Expected to convert into a scalable vector!");
2710 assert(V.getValueType().isFixedLengthVector() &&
2711 "Expected a fixed length vector operand!");
2721 "Expected to convert into a fixed length vector!");
2722 assert(V.getValueType().isScalableVector() &&
2723 "Expected a scalable vector operand!");
2746static std::pair<SDValue, SDValue>
2755static std::pair<SDValue, SDValue>
2768static std::pair<SDValue, SDValue>
2785std::pair<unsigned, unsigned>
2801 return std::make_pair(MinVLMAX, MaxVLMAX);
2813 EVT VT,
unsigned DefinedValues)
const {
2827 std::tie(LMul, Fractional) =
2830 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2832 Cost = (LMul * DLenFactor);
2875 Op.getValueType() == MVT::bf16) {
2876 bool IsStrict =
Op->isStrictFPOpcode();
2881 {Op.getOperand(0), Op.getOperand(1)});
2883 {
Op.getValueType(), MVT::Other},
2889 DAG.
getNode(
Op.getOpcode(),
DL, MVT::f32,
Op.getOperand(0)),
2904 MVT DstVT =
Op.getSimpleValueType();
2905 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2913 Src.getValueType() == MVT::bf16) {
2920 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2928 Opc,
DL, DstVT, Src,
2942 MVT SrcVT = Src.getSimpleValueType();
2948 if (SatVT != DstEltVT)
2951 MVT DstContainerVT = DstVT;
2952 MVT SrcContainerVT = SrcVT;
2958 "Expected same element count");
2967 {Src, Src, DAG.getCondCode(ISD::SETNE),
2968 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2972 if (DstEltSize > (2 * SrcEltSize)) {
2978 MVT CvtContainerVT = DstContainerVT;
2979 MVT CvtEltVT = DstEltVT;
2980 if (SrcEltSize > (2 * DstEltSize)) {
2989 while (CvtContainerVT != DstContainerVT) {
2995 Res = DAG.
getNode(ClipOpc,
DL, CvtContainerVT, Res, Mask, VL);
3002 Res, DAG.
getUNDEF(DstContainerVT), VL);
3012 bool IsStrict =
Op->isStrictFPOpcode();
3013 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
3023 {
Op.getOperand(0), SrcVal});
3024 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
3025 {Ext.getValue(1), Ext.getValue(0)});
3039 case ISD::VP_FROUNDEVEN:
3043 case ISD::VP_FROUNDTOZERO:
3047 case ISD::VP_FFLOOR:
3055 case ISD::VP_FROUND:
3072 MVT VT =
Op.getSimpleValueType();
3079 MVT ContainerVT = VT;
3086 if (
Op->isVPOpcode()) {
3087 Mask =
Op.getOperand(1);
3091 VL =
Op.getOperand(2);
3113 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3127 switch (
Op.getOpcode()) {
3135 case ISD::VP_FFLOOR:
3138 case ISD::VP_FROUND:
3139 case ISD::VP_FROUNDEVEN:
3140 case ISD::VP_FROUNDTOZERO: {
3152 case ISD::VP_FNEARBYINT:
3165 Src, Src, Mask, VL);
3180 MVT VT =
Op.getSimpleValueType();
3184 MVT ContainerVT = VT;
3196 MVT MaskVT = Mask.getSimpleValueType();
3199 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3200 DAG.getUNDEF(MaskVT), Mask, VL});
3204 {Chain, Src, Src, Src, Unorder, VL});
3205 Chain = Src.getValue(1);
3221 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3233 switch (
Op.getOpcode()) {
3244 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3250 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3254 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3263 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3264 Truncated, Mask, VL);
3270 Src, Src, Mask, VL);
3280 MVT VT =
Op.getSimpleValueType();
3308 MVT VT =
Op.getSimpleValueType();
3313 MVT ContainerVT = VT;
3358 "Unexpected vector MVT");
3386 return std::nullopt;
3404 unsigned EltSizeInBits) {
3407 return std::nullopt;
3408 bool IsInteger =
Op.getValueType().isInteger();
3410 std::optional<unsigned> SeqStepDenom;
3411 std::optional<APInt> SeqStepNum;
3412 std::optional<APInt> SeqAddend;
3413 std::optional<std::pair<APInt, unsigned>> PrevElt;
3414 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3419 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3421 if (Elt.isUndef()) {
3422 Elts[
Idx] = std::nullopt;
3426 Elts[
Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
3431 return std::nullopt;
3432 Elts[
Idx] = *ExactInteger;
3445 unsigned IdxDiff =
Idx - PrevElt->second;
3446 APInt ValDiff = *Elt - PrevElt->first;
3454 int64_t Remainder = ValDiff.
srem(IdxDiff);
3459 return std::nullopt;
3460 ValDiff = ValDiff.
sdiv(IdxDiff);
3465 SeqStepNum = ValDiff;
3466 else if (ValDiff != SeqStepNum)
3467 return std::nullopt;
3470 SeqStepDenom = IdxDiff;
3471 else if (IdxDiff != *SeqStepDenom)
3472 return std::nullopt;
3476 if (!PrevElt || PrevElt->first != *Elt)
3477 PrevElt = std::make_pair(*Elt,
Idx);
3481 if (!SeqStepNum || !SeqStepDenom)
3482 return std::nullopt;
3490 (
APInt(EltSizeInBits,
Idx,
false,
true) *
3492 .sdiv(*SeqStepDenom);
3494 APInt Addend = *Elt - ExpectedVal;
3497 else if (Addend != SeqAddend)
3498 return std::nullopt;
3501 assert(SeqAddend &&
"Must have an addend if we have a step");
3503 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
3504 SeqAddend->getSExtValue()};
3519 if (EltTy == MVT::i1 ||
3532 MVT ContainerVT = VT;
3561 MVT VT =
Op.getSimpleValueType();
3573 unsigned MostCommonCount = 0;
3575 unsigned NumUndefElts =
3583 unsigned NumScalarLoads = 0;
3589 unsigned &Count = ValueCounts[V];
3591 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3592 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3597 if (++Count >= MostCommonCount) {
3599 MostCommonCount = Count;
3603 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3604 unsigned NumDefElts = NumElts - NumUndefElts;
3605 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3611 ((MostCommonCount > DominantValueCountThreshold) ||
3624 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3625 LastOp != DominantValue) {
3634 Processed.insert(LastOp);
3639 const SDValue &V = OpIdx.value();
3640 if (V.isUndef() || !Processed.insert(V).second)
3642 if (ValueCounts[V] == 1) {
3651 return DAG.getConstant(V == V1, DL, XLenVT);
3667 MVT VT =
Op.getSimpleValueType();
3697 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3698 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3706 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3707 MVT IntegerViaVecVT =
3712 unsigned BitPos = 0, IntegerEltIdx = 0;
3715 for (
unsigned I = 0;
I < NumElts;) {
3717 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3718 Bits |= ((
uint64_t)BitValue << BitPos);
3724 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3725 if (NumViaIntegerBits <= 32)
3726 Bits = SignExtend64<32>(Bits);
3728 Elts[IntegerEltIdx] = Elt;
3737 if (NumElts < NumViaIntegerBits) {
3741 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3769 int64_t StepNumerator = SimpleVID->StepNumerator;
3770 unsigned StepDenominator = SimpleVID->StepDenominator;
3771 int64_t Addend = SimpleVID->Addend;
3773 assert(StepNumerator != 0 &&
"Invalid step");
3774 bool Negate =
false;
3775 int64_t SplatStepVal = StepNumerator;
3779 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3781 Negate = StepNumerator < 0;
3783 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3790 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3791 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3793 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3796 MVT VIDContainerVT =
3804 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3805 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3807 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3809 if (StepDenominator != 1) {
3814 if (Addend != 0 || Negate) {
3833 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3834 "Unexpected sequence type");
3838 unsigned ViaVecLen =
3842 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3845 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3846 const auto &SeqV = OpIdx.value();
3847 if (!SeqV.isUndef())
3849 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3855 if (ViaIntVT == MVT::i32)
3856 SplatValue = SignExtend64<32>(SplatValue);
3878 const auto *BV = cast<BuildVectorSDNode>(
Op);
3881 BV->getRepeatedSequence(Sequence) &&
3882 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3883 unsigned SeqLen = Sequence.size();
3885 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3886 ViaIntVT == MVT::i64) &&
3887 "Unexpected sequence type");
3892 const unsigned RequiredVL = NumElts / SeqLen;
3893 const unsigned ViaVecLen =
3895 NumElts : RequiredVL;
3898 unsigned EltIdx = 0;
3899 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3903 for (
const auto &SeqV : Sequence) {
3904 if (!SeqV.isUndef())
3906 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3913 if (ViaIntVT == MVT::i32)
3914 SplatValue = SignExtend64<32>(SplatValue);
3920 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3921 "Unexpected bitcast sequence");
3922 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3925 MVT ViaContainerVT =
3932 if (ViaVecLen != RequiredVL)
3952 Source, DAG, Subtarget);
3972 return RISCV::PACKH;
3974 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
3989 MVT VT =
Op.getSimpleValueType();
3997 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4002 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
4016 if (Subtarget.hasStdExtZbkb())
4021 ElemDL, XLenVT,
A,
B),
4033 NewOperands.
reserve(NumElts / 2);
4035 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
4045 MVT VT =
Op.getSimpleValueType();
4056 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) || EltVT == MVT::bf16) {
4061 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4062 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4065 if (
auto *
C = dyn_cast<ConstantFPSDNode>(Elem)) {
4149 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
4153 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4176 unsigned NumUndefElts =
4178 unsigned NumDefElts = NumElts - NumUndefElts;
4179 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4186 for (
unsigned i = 0; i < NumElts; i++) {
4188 if (i < NumElts / 2) {
4195 bool SelectMaskVal = (i < NumElts / 2);
4198 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4199 MaskVals.
size() == NumElts);
4234 unsigned UndefCount = 0;
4241 LinearBudget -= PerSlideCost;
4244 LinearBudget -= PerSlideCost;
4247 LinearBudget -= PerSlideCost;
4250 if (LinearBudget < 0)
4255 "Illegal type which will result in reserved encoding");
4280 Vec,
Offset, Mask, VL, Policy);
4293 Vec,
Offset, Mask, VL, Policy);
4303 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4304 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4305 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4308 if ((LoC >> 31) == HiC)
4319 (isa<RegisterSDNode>(VL) &&
4320 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4322 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4337 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4338 Hi.getConstantOperandVal(1) == 31)
4357 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4369 bool HasPassthru = Passthru && !Passthru.
isUndef();
4370 if (!HasPassthru && !Passthru)
4377 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
4378 EltVT == MVT::bf16) {
4379 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4380 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
4394 if (Scalar.getValueType().bitsLE(XLenVT)) {
4401 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4405 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4406 "Unexpected scalar for splat lowering!");
4430 SDValue ExtractedVal = Scalar.getOperand(0);
4435 MVT ExtractedContainerVT = ExtractedVT;
4438 DAG, ExtractedContainerVT, Subtarget);
4440 ExtractedVal, DAG, Subtarget);
4442 if (ExtractedContainerVT.
bitsLE(VT))
4457 if (!Scalar.getValueType().bitsLE(XLenVT))
4460 VT,
DL, DAG, Subtarget);
4468 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4488 if (Src != V2.getOperand(0))
4493 if (Src.getValueType().getVectorNumElements() != (NumElts * 2))
4498 V2.getConstantOperandVal(1) != NumElts)
4516 int Size = Mask.size();
4518 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4524 EvenSrc = StartIndexes[0];
4525 OddSrc = StartIndexes[1];
4528 if (EvenSrc != 0 && OddSrc != 0)
4538 int HalfNumElts = NumElts / 2;
4539 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4555 int Size = Mask.size();
4567 for (
int i = 0; i !=
Size; ++i) {
4573 int StartIdx = i - (M %
Size);
4581 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4584 Rotation = CandidateRotation;
4585 else if (Rotation != CandidateRotation)
4590 int MaskSrc = M <
Size ? 0 : 1;
4595 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4600 TargetSrc = MaskSrc;
4601 else if (TargetSrc != MaskSrc)
4608 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4609 assert((LoSrc >= 0 || HiSrc >= 0) &&
4610 "Failed to find a rotated input vector!");
4624 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
4631 unsigned Shift = Index * EltBits;
4658 auto findNonEXTRACT_SUBVECTORParent =
4659 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4664 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4665 Offset += Parent.getConstantOperandVal(1);
4666 Parent = Parent.getOperand(0);
4668 return std::make_pair(Parent,
Offset);
4671 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4672 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4681 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4682 if (NewMask[i] == -1)
4685 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4686 NewMask[i] = NewMask[i] + V1IndexOffset;
4690 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4696 if (NewMask[0] <= 0)
4700 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4701 if (NewMask[i - 1] + 1 != NewMask[i])
4705 MVT SrcVT = Src.getSimpleValueType();
4736 int NumSubElts, Index;
4741 bool OpsSwapped = Mask[Index] < (int)NumElts;
4742 SDValue InPlace = OpsSwapped ? V2 : V1;
4743 SDValue ToInsert = OpsSwapped ? V1 : V2;
4753 if (NumSubElts + Index >= (
int)NumElts)
4767 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4779 bool OpsSwapped =
false;
4780 if (!isa<BuildVectorSDNode>(V1)) {
4781 if (!isa<BuildVectorSDNode>(V2))
4786 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4794 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4795 for (
unsigned i = S; i != E; ++i)
4796 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4802 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4803 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4806 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4808 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4831 auto OpCode = IsVSlidedown ?
4836 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4839 Splat, TrueMask, VL);
4850 for (
unsigned i = 0; i < Mask.size(); i++)
4851 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
4854 for (
unsigned i = 0; i < Factor; i++) {
4865 for (
unsigned i = 0; i < Mask.size() / Factor; i++) {
4866 unsigned j = i * Factor + Index;
4867 if (Mask[j] != -1 && (
unsigned)Mask[j] != i)
4880 MVT VT = V.getSimpleValueType();
4895 EC.multiplyCoefficientBy(Factor));
4914 MVT VecContainerVT = VecVT;
4931 MVT WideContainerVT = WideVT;
4937 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4944 if (Subtarget.hasStdExtZvbb()) {
4949 OffsetVec, Passthru, Mask, VL);
4951 Interleaved, EvenV, Passthru, Mask, VL);
4959 OddV, Passthru, Mask, VL);
4965 OddV, AllOnesVec, Passthru, Mask, VL);
4973 Interleaved, OddsMul, Passthru, Mask, VL);
4980 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
5026 if (ViaEltSize > NumElts)
5035 if (ViaEltSize > NumElts)
5041 if (ViaEltSize > NumElts)
5050 MVT &RotateVT,
unsigned &RotateAmt) {
5056 unsigned NumSubElts;
5058 NumElts, NumSubElts, RotateAmt))
5061 NumElts / NumSubElts);
5123 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
5126 OutMasks(VRegsPerSrc, {-1, {}});
5131 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
5132 int DstVecIdx = DstIdx / ElemsPerVReg;
5133 int DstSubIdx = DstIdx % ElemsPerVReg;
5134 int SrcIdx = Mask[DstIdx];
5135 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
5137 int SrcVecIdx = SrcIdx / ElemsPerVReg;
5138 int SrcSubIdx = SrcIdx % ElemsPerVReg;
5139 if (OutMasks[DstVecIdx].first == -1)
5140 OutMasks[DstVecIdx].first = SrcVecIdx;
5141 if (OutMasks[DstVecIdx].first != SrcVecIdx)
5147 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
5148 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
5162 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
5163 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
5164 if (SrcVecIdx == -1)
5166 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
5173 unsigned InsertIdx = DstVecIdx * NumOpElts;
5185 bool SawUndef =
false;
5186 for (
unsigned i = 0; i < Mask.size(); i++) {
5187 if (Mask[i] == -1) {
5193 if (i > (
unsigned)Mask[i])
5195 if (Mask[i] <=
Last)
5226 for (
int Idx : Mask) {
5229 unsigned SrcIdx =
Idx % Mask.size();
5231 if (Srcs[SrcIdx] == -1)
5234 else if (Srcs[SrcIdx] != Src)
5240 for (
int Lane : Srcs) {
5253 for (
unsigned I = 0;
I < Mask.size();
I++) {
5257 NewMask[
I] = Mask[
I] % Mask.size();
5269 MVT VT =
Op.getSimpleValueType();
5284 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5308 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5309 V = V.getOperand(
Offset / OpElements);
5315 auto *Ld = cast<LoadSDNode>(V);
5325 SDValue Ops[] = {Ld->getChain(),
5339 MVT SplatVT = ContainerVT;
5342 if (SVT == MVT::bf16 ||
5343 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
5352 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5353 Ld->getPointerInfo().getWithOffset(
Offset),
5354 Ld->getOriginalAlign(),
5358 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5359 Ld->getOriginalAlign(),
5360 Ld->getMemOperand()->getFlags());
5372 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5375 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5397 if (Subtarget.hasStdExtZvkb())
5408 LoV = LoSrc == 0 ? V1 : V2;
5412 HiV = HiSrc == 0 ? V1 : V2;
5418 unsigned InvRotate = NumElts - Rotation;
5428 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5446 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
5447 for (
unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
5463 int EvenSrc, OddSrc;
5471 bool LaneIsUndef[2] = {
true,
true};
5472 for (
unsigned i = 0; i < Mask.size(); i++)
5473 LaneIsUndef[i % 2] &= (Mask[i] == -1);
5475 int Size = Mask.size();
5477 if (LaneIsUndef[0]) {
5480 assert(EvenSrc >= 0 &&
"Undef source?");
5481 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5486 if (LaneIsUndef[1]) {
5489 assert(OddSrc >= 0 &&
"Undef source?");
5490 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5500 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5515 for (
auto Idx : Mask) {
5531 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
5532 for (
unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
5545 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5574 MVT IndexContainerVT =
5579 for (
int MaskIndex : Mask) {
5580 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5589 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5599 for (
int MaskIndex : Mask) {
5600 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5601 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5603 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5629 for (
int MaskIndex : Mask) {
5630 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5634 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5666RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5668 MVT VT =
Op.getSimpleValueType();
5672 MVT ContainerVT = VT;
5675 if (
Op->isVPOpcode()) {
5676 Mask =
Op.getOperand(1);
5680 VL =
Op.getOperand(2);
5686 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5688 FloatEltVT = MVT::f32;
5695 "Expected legal float type!");
5702 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5705 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5710 if (FloatVT.
bitsGT(VT)) {
5711 if (
Op->isVPOpcode())
5712 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5721 if (!
Op->isVPOpcode())
5725 MVT ContainerFloatVT =
5728 Src, Mask, RTZRM, VL);
5735 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5739 if (
Op->isVPOpcode()) {
5748 else if (IntVT.
bitsGT(VT))
5753 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5758 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5759 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5764 unsigned Adjust = ExponentBias + (EltSize - 1);
5766 if (
Op->isVPOpcode())
5776 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5777 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5787 MVT SrcVT =
Source.getSimpleValueType();
5796 SrcVT = ContainerVT;
5809 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5826 auto *
Load = cast<LoadSDNode>(
Op);
5827 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5830 Load->getMemoryVT(),
5831 *
Load->getMemOperand()))
5835 MVT VT =
Op.getSimpleValueType();
5837 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5838 "Unexpected unaligned RVV load type");
5842 "Expecting equally-sized RVV vector types to be legal");
5844 Load->getPointerInfo(),
Load->getOriginalAlign(),
5845 Load->getMemOperand()->getFlags());
5855 auto *
Store = cast<StoreSDNode>(
Op);
5856 assert(Store &&
Store->getValue().getValueType().isVector() &&
5857 "Expected vector store");
5860 Store->getMemoryVT(),
5861 *
Store->getMemOperand()))
5868 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5869 "Unexpected unaligned RVV store type");
5873 "Expecting equally-sized RVV vector types to be legal");
5874 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5876 Store->getPointerInfo(),
Store->getOriginalAlign(),
5877 Store->getMemOperand()->getFlags());
5882 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5884 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5911 unsigned ShiftAmt, AddOpc;
5922 MVT VT =
Op.getSimpleValueType();
5923 const APFloat &
Imm = cast<ConstantFPSDNode>(
Op)->getValueAPF();
5926 bool Negate =
false;
5930 if (Index < 0 &&
Imm.isNegative()) {
5958 if (Subtarget.hasStdExtZtso()) {
5982 MVT VT =
Op.getSimpleValueType();
5984 unsigned Check =
Op.getConstantOperandVal(1);
5985 unsigned TDCMask = 0;
6013 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
6018 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
6020 VL =
Op.getOperand(3);
6023 VL,
Op->getFlags());
6038 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
6040 MVT MaskContainerVT =
6043 VL =
Op.getOperand(3);
6048 Mask, VL,
Op->getFlags());
6051 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
6056 DAG.
getUNDEF(ContainerVT), Mask, VL});
6060 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
6064 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
6068 DAG.
getUNDEF(ContainerVT), Mask, VL});
6084 MVT VT =
Op.getSimpleValueType();
6111 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
6118 MVT ContainerVT = VT;
6126 if (
Op->isVPOpcode()) {
6127 Mask =
Op.getOperand(2);
6131 VL =
Op.getOperand(3);
6139 {X, X, DAG.getCondCode(ISD::SETOEQ),
6140 DAG.getUNDEF(ContainerVT), Mask, VL});
6148 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
6149 DAG.getUNDEF(ContainerVT), Mask, VL});
6159 DAG.
getUNDEF(ContainerVT), Mask, VL);
6169 "Wrong opcode for lowering FABS or FNEG.");
6172 MVT VT =
Op.getSimpleValueType();
6173 assert((VT == MVT::f16 || VT == MVT::bf16) &&
"Unexpected type");
6180 Mask = Mask.sext(Subtarget.
getXLen());
6193 MVT VT =
Op.getSimpleValueType();
6194 assert((VT == MVT::f16 || VT == MVT::bf16) &&
"Unexpected type");
6204 if (SignSize == Subtarget.
getXLen()) {
6206 }
else if (SignSize == 16) {
6208 }
else if (SignSize == 32) {
6210 }
else if (SignSize == 64) {
6211 assert(XLenVT == MVT::i32 &&
"Unexpected type");
6221 if (ShiftAmount > 0) {
6224 }
else if (ShiftAmount < 0) {
6250#define OP_CASE(NODE) \
6252 return RISCVISD::NODE##_VL;
6253#define VP_CASE(NODE) \
6254 case ISD::VP_##NODE: \
6255 return RISCVISD::NODE##_VL;
6257 switch (
Op.getOpcode()) {
6335 case ISD::VP_CTLZ_ZERO_UNDEF:
6338 case ISD::VP_CTTZ_ZERO_UNDEF:
6347 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6352 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6357 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6360 case ISD::VP_SELECT:
6369 case ISD::VP_SIGN_EXTEND:
6371 case ISD::VP_ZERO_EXTEND:
6373 case ISD::VP_FP_TO_SINT:
6375 case ISD::VP_FP_TO_UINT:
6378 case ISD::VP_FMINNUM:
6381 case ISD::VP_FMAXNUM:
6386 case ISD::VP_LLRINT:
6398 "not a RISC-V target specific op");
6404 "adding target specific op should update this function");
6424 "not a RISC-V target specific op");
6430 "adding target specific op should update this function");
6443 if (
Op.getValueType() == MVT::nxv32f16 &&
6447 if (
Op.getValueType() == MVT::nxv32bf16)
6460 if (!
Op.getOperand(j).getValueType().isVector()) {
6461 LoOperands[j] =
Op.getOperand(j);
6462 HiOperands[j] =
Op.getOperand(j);
6465 std::tie(LoOperands[j], HiOperands[j]) =
6470 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6472 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6487 std::tie(LoOperands[j], HiOperands[j]) =
6491 if (!
Op.getOperand(j).getValueType().isVector()) {
6492 LoOperands[j] =
Op.getOperand(j);
6493 HiOperands[j] =
Op.getOperand(j);
6496 std::tie(LoOperands[j], HiOperands[j]) =
6501 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6503 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6513 auto [EVLLo, EVLHi] =
6514 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6518 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6520 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6538 if (!
Op.getOperand(j).getValueType().isVector()) {
6539 LoOperands[j] =
Op.getOperand(j);
6540 HiOperands[j] =
Op.getOperand(j);
6543 std::tie(LoOperands[j], HiOperands[j]) =
6548 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6551 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6560 switch (
Op.getOpcode()) {
6566 return lowerGlobalAddress(
Op, DAG);
6568 return lowerBlockAddress(
Op, DAG);
6570 return lowerConstantPool(
Op, DAG);
6572 return lowerJumpTable(
Op, DAG);
6574 return lowerGlobalTLSAddress(
Op, DAG);
6578 return lowerConstantFP(
Op, DAG);
6580 return lowerSELECT(
Op, DAG);
6582 return lowerBRCOND(
Op, DAG);
6584 return lowerVASTART(
Op, DAG);
6586 return lowerFRAMEADDR(
Op, DAG);
6588 return lowerRETURNADDR(
Op, DAG);
6590 return lowerShiftLeftParts(
Op, DAG);
6592 return lowerShiftRightParts(
Op, DAG,
true);
6594 return lowerShiftRightParts(
Op, DAG,
false);
6597 if (
Op.getValueType().isFixedLengthVector()) {
6598 assert(Subtarget.hasStdExtZvkb());
6599 return lowerToScalableOp(
Op, DAG);
6601 assert(Subtarget.hasVendorXTHeadBb() &&
6602 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6603 "Unexpected custom legalization");
6605 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6610 EVT VT =
Op.getValueType();
6614 if (Op0VT == MVT::i16 &&
6616 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
6620 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6625 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.
is64Bit() &&
6641 "Unexpected types");
6675 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6677 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6679 return LowerINTRINSIC_VOID(
Op, DAG);
6681 return LowerIS_FPCLASS(
Op, DAG);
6683 MVT VT =
Op.getSimpleValueType();
6685 assert(Subtarget.hasStdExtZvbb());
6686 return lowerToScalableOp(
Op, DAG);
6689 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6699 if (!
Op.getSimpleValueType().isVector())
6701 return lowerVectorTruncLike(
Op, DAG);
6704 if (
Op.getOperand(0).getValueType().isVector() &&
6705 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6706 return lowerVectorMaskExt(
Op, DAG, 1);
6709 if (
Op.getOperand(0).getValueType().isVector() &&
6710 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6711 return lowerVectorMaskExt(
Op, DAG, -1);
6714 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6716 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6718 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6720 MVT VT =
Op.getSimpleValueType();
6728 MVT ContainerVT = VT;
6736 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6740 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6748 MVT VT =
Op.getSimpleValueType();
6768 }
else if ((Val % 8) == 0) {
6784 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6785 Op.getOperand(1).getValueType() == MVT::i32) {
6802 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6805 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6808 if (
Op.getValueType().isVector() &&
6809 ((
Op.getValueType().getScalarType() == MVT::f16 &&
6812 Op.getValueType().getScalarType() == MVT::bf16)) {
6828 Op1.getValueType().isVector() &&
6829 ((Op1.getValueType().getScalarType() == MVT::f16 &&
6832 Op1.getValueType().getScalarType() == MVT::bf16)) {
6838 Op1.getValueType().getVectorElementCount());
6841 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6851 MVT VT =
Op.getSimpleValueType();
6854 bool IsStrict =
Op->isStrictFPOpcode();
6855 SDValue Src =
Op.getOperand(0 + IsStrict);
6856 MVT SrcVT = Src.getSimpleValueType();
6867 "Unexpected vector element types");
6871 if (EltSize > (2 * SrcEltSize)) {
6883 Op.getOperand(0), Ext);
6887 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6892 auto [FExt, Chain] =
6894 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6901 if (SrcEltSize > (2 * EltSize)) {
6904 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6909 Op.getOperand(0), Src);
6924 Op.getOperand(0), Src);
6938 unsigned RVVOpc = 0;
6939 switch (
Op.getOpcode()) {
6971 "Expected same element count");
6978 Op.getOperand(0), Src, Mask, VL);
6982 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6997 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
7004 MVT VT =
Op.getSimpleValueType();
7024 bool IsStrict =
Op->isStrictFPOpcode();
7025 SDValue Op0 = IsStrict ?
Op.getOperand(1) :
Op.getOperand(0);
7029 std::tie(Res, Chain) =
7030 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions,
DL, Chain);
7045 bool IsStrict =
Op->isStrictFPOpcode();
7046 SDValue Op0 = IsStrict ?
Op.getOperand(1) :
Op.getOperand(0);
7052 std::tie(Res, Chain) =
makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
7053 CallOptions,
DL, Chain);
7070 if (
Op.getValueType().isVector())
7075 assert(
Op.getOperand(0).getValueType() == MVT::f16 &&
7076 "Unexpected custom legalisation");
7079 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), Ext);
7085 assert(
Op.getOperand(1).getValueType() == MVT::f16 &&
7086 "Unexpected custom legalisation");
7089 {
Op.getOperand(0),
Op.getOperand(1)});
7090 return DAG.
getNode(
Op.getOpcode(),
DL, {Op.getValueType(), MVT::Other},
7091 {Ext.getValue(1), Ext.getValue(0)});
7098 return lowerVECREDUCE(
Op, DAG);
7102 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
7103 return lowerVectorMaskVecReduction(
Op, DAG,
false);
7104 return lowerVECREDUCE(
Op, DAG);
7111 return lowerFPVECREDUCE(
Op, DAG);
7112 case ISD::VP_REDUCE_ADD:
7113 case ISD::VP_REDUCE_UMAX:
7114 case ISD::VP_REDUCE_SMAX:
7115 case ISD::VP_REDUCE_UMIN:
7116 case ISD::VP_REDUCE_SMIN:
7117 case ISD::VP_REDUCE_FADD:
7118 case ISD::VP_REDUCE_SEQ_FADD:
7119 case ISD::VP_REDUCE_FMIN:
7120 case ISD::VP_REDUCE_FMAX:
7121 case ISD::VP_REDUCE_FMINIMUM:
7122 case ISD::VP_REDUCE_FMAXIMUM:
7125 return lowerVPREDUCE(
Op, DAG);
7126 case ISD::VP_REDUCE_AND:
7127 case ISD::VP_REDUCE_OR:
7128 case ISD::VP_REDUCE_XOR:
7129 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
7130 return lowerVectorMaskVecReduction(
Op, DAG,
true);
7131 return lowerVPREDUCE(
Op, DAG);
7132 case ISD::VP_CTTZ_ELTS:
7133 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
7134 return lowerVPCttzElements(
Op, DAG);
7138 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
7141 return lowerINSERT_SUBVECTOR(
Op, DAG);
7143 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
7145 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
7147 return lowerVECTOR_INTERLEAVE(
Op, DAG);
7149 return lowerSTEP_VECTOR(
Op, DAG);
7151 return lowerVECTOR_REVERSE(
Op, DAG);
7153 return lowerVECTOR_SPLICE(
Op, DAG);
7157 MVT VT =
Op.getSimpleValueType();
7159 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
7160 EltVT == MVT::bf16) {
7163 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
7164 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
7174 if (EltVT == MVT::i1)
7175 return lowerVectorMaskSplat(
Op, DAG);
7184 MVT VT =
Op.getSimpleValueType();
7185 MVT ContainerVT = VT;
7203 Op->ops().take_front(HalfNumOps));
7205 Op->ops().drop_front(HalfNumOps));
7209 unsigned NumOpElts =
7210 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
7213 SDValue SubVec = OpIdx.value();
7224 auto *Load = cast<LoadSDNode>(
Op);
7225 EVT VecTy = Load->getMemoryVT();
7232 unsigned NumElts = Sz / (NF * 8);
7233 int Log2LMUL =
Log2_64(NumElts) - 3;
7236 Flag.setNoUnsignedWrap(
true);
7238 SDValue BasePtr = Load->getBasePtr();
7246 for (
unsigned i = 0; i < NF; ++i) {
7259 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
7261 if (
Op.getValueType().isFixedLengthVector())
7262 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
7266 auto *Store = cast<StoreSDNode>(
Op);
7267 SDValue StoredVal = Store->getValue();
7275 unsigned NumElts = Sz / (NF * 8);
7276 int Log2LMUL =
Log2_64(NumElts) - 3;
7279 Flag.setNoUnsignedWrap(
true);
7281 SDValue Chain = Store->getChain();
7282 SDValue BasePtr = Store->getBasePtr();
7289 for (
unsigned i = 0; i < NF; ++i) {
7293 Ret = DAG.
getStore(Chain,
DL, Extract, BasePtr,
7295 Store->getOriginalAlign(),
7296 Store->getMemOperand()->getFlags());
7297 Chain = Ret.getValue(0);
7303 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
7305 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
7306 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
7311 return lowerMaskedLoad(
Op, DAG);
7314 return lowerMaskedStore(
Op, DAG);
7316 return lowerVectorCompress(
Op, DAG);
7325 EVT VT =
Op.getValueType();
7336 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
7338 MVT VT =
Op.getSimpleValueType();
7343 "Unexpected CondCode");
7351 if (isa<ConstantSDNode>(
RHS)) {
7352 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
7353 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
7375 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
7391 return lowerToScalableOp(
Op, DAG);
7395 if (
Op.getSimpleValueType().isFixedLengthVector())
7396 return lowerToScalableOp(
Op, DAG);
7398 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
7399 "Unexpected custom legalisation");
7403 if (
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16)
7429 return lowerToScalableOp(
Op, DAG);
7433 EVT VT =
Op->getValueType(0);
7448 return lowerABS(
Op, DAG);
7453 if (Subtarget.hasStdExtZvbb())
7454 return lowerToScalableOp(
Op, DAG);
7456 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7458 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
7460 if (
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16)
7464 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
7473 return lowerToScalableOp(
Op, DAG);
7476 return lowerVectorStrictFSetcc(
Op, DAG);
7486 case ISD::VP_GATHER:
7487 return lowerMaskedGather(
Op, DAG);
7489 case ISD::VP_SCATTER:
7490 return lowerMaskedScatter(
Op, DAG);
7492 return lowerGET_ROUNDING(
Op, DAG);
7494 return lowerSET_ROUNDING(
Op, DAG);
7496 return lowerEH_DWARF_CFA(
Op, DAG);
7498 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7499 return lowerVPMergeMask(
Op, DAG);
7501 case ISD::VP_SELECT:
7509 case ISD::VP_UADDSAT:
7510 case ISD::VP_USUBSAT:
7511 case ISD::VP_SADDSAT:
7512 case ISD::VP_SSUBSAT:
7514 case ISD::VP_LLRINT:
7515 return lowerVPOp(
Op, DAG);
7519 return lowerLogicVPOp(
Op, DAG);
7528 case ISD::VP_FMINNUM:
7529 case ISD::VP_FMAXNUM:
7530 case ISD::VP_FCOPYSIGN:
7537 return lowerVPOp(
Op, DAG);
7538 case ISD::VP_IS_FPCLASS:
7539 return LowerIS_FPCLASS(
Op, DAG);
7540 case ISD::VP_SIGN_EXTEND:
7541 case ISD::VP_ZERO_EXTEND:
7542 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7543 return lowerVPExtMaskOp(
Op, DAG);
7544 return lowerVPOp(
Op, DAG);
7545 case ISD::VP_TRUNCATE:
7546 return lowerVectorTruncLike(
Op, DAG);
7547 case ISD::VP_FP_EXTEND:
7548 case ISD::VP_FP_ROUND:
7549 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7550 case ISD::VP_SINT_TO_FP:
7551 case ISD::VP_UINT_TO_FP:
7552 if (
Op.getValueType().isVector() &&
7553 ((
Op.getValueType().getScalarType() == MVT::f16 &&
7556 Op.getValueType().getScalarType() == MVT::bf16)) {
7569 case ISD::VP_FP_TO_SINT:
7570 case ISD::VP_FP_TO_UINT:
7572 Op1.getValueType().isVector() &&
7573 ((Op1.getValueType().getScalarType() == MVT::f16 &&
7576 Op1.getValueType().getScalarType() == MVT::bf16)) {
7582 Op1.getValueType().getVectorElementCount());
7586 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7588 return lowerVPFPIntConvOp(
Op, DAG);
7592 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7593 return lowerVPSetCCMaskOp(
Op, DAG);
7599 case ISD::VP_BITREVERSE:
7601 return lowerVPOp(
Op, DAG);
7603 case ISD::VP_CTLZ_ZERO_UNDEF:
7604 if (Subtarget.hasStdExtZvbb())
7605 return lowerVPOp(
Op, DAG);
7606 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7608 case ISD::VP_CTTZ_ZERO_UNDEF:
7609 if (Subtarget.hasStdExtZvbb())
7610 return lowerVPOp(
Op, DAG);
7611 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7613 return lowerVPOp(
Op, DAG);
7614 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7615 return lowerVPStridedLoad(
Op, DAG);
7616 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7617 return lowerVPStridedStore(
Op, DAG);
7619 case ISD::VP_FFLOOR:
7621 case ISD::VP_FNEARBYINT:
7622 case ISD::VP_FROUND:
7623 case ISD::VP_FROUNDEVEN:
7624 case ISD::VP_FROUNDTOZERO:
7628 case ISD::VP_FMAXIMUM:
7629 case ISD::VP_FMINIMUM:
7633 case ISD::EXPERIMENTAL_VP_SPLICE:
7634 return lowerVPSpliceExperimental(
Op, DAG);
7635 case ISD::EXPERIMENTAL_VP_REVERSE:
7636 return lowerVPReverseExperimental(
Op, DAG);
7637 case ISD::EXPERIMENTAL_VP_SPLAT:
7638 return lowerVPSplatExperimental(
Op, DAG);
7641 "llvm.clear_cache only needs custom lower on Linux targets");
7644 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7645 Op.getOperand(2), Flags,
DL);
7648 return lowerINIT_TRAMPOLINE(
Op, DAG);
7650 return lowerADJUST_TRAMPOLINE(
Op, DAG);
7657 MakeLibCallOptions CallOptions;
7658 std::pair<SDValue, SDValue> CallResult =
7659 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7660 {Start,
End, Flags}, CallOptions,
DL, InChain);
7663 return CallResult.second;
7676 std::unique_ptr<MCCodeEmitter> CodeEmitter(
7683 const Value *TrmpAddr = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
7695 constexpr unsigned StaticChainOffset = 16;
7696 constexpr unsigned FunctionAddressOffset = 24;
7700 auto GetEncoding = [&](
const MCInst &MC) {
7703 CodeEmitter->encodeInstruction(MC, CB, Fixups, *STI);
7713 GetEncoding(
MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X7).addImm(0)),
7718 MCInstBuilder(RISCV::LD).addReg(RISCV::X5).addReg(RISCV::X7).addImm(
7719 FunctionAddressOffset)),
7723 MCInstBuilder(RISCV::LD).addReg(RISCV::X7).addReg(RISCV::X7).addImm(
7724 StaticChainOffset)),
7743 SDValue FunctionAddress =
Op.getOperand(2);
7747 struct OffsetValuePair {
7751 } OffsetValues[] = {
7752 {StaticChainOffset, StaticChain},
7753 {FunctionAddressOffset, FunctionAddress},
7758 DAG.
getConstant(OffsetValue.Offset, dl, MVT::i64));
7759 OffsetValue.Addr =
Addr;
7760 OutChains[
Idx + 4] =
7769 SDValue EndOfTrmp = OffsetValues[0].Addr;
7783 return Op.getOperand(0);
7800 N->getOffset(), Flags);
7829template <
class NodeTy>
7831 bool IsLocal,
bool IsExternWeak)
const {
7841 if (IsLocal && !Subtarget.allowTaggedGlobals())
7911 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7920 return getAddr(
N, DAG);
7927 return getAddr(
N, DAG);
7934 return getAddr(
N, DAG);
7939 bool UseGOT)
const {
8003 Args.push_back(Entry);
8036 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
8050 Addr = getStaticTLSAddr(
N, DAG,
false);
8053 Addr = getStaticTLSAddr(
N, DAG,
true);
8058 : getDynamicTLSAddr(
N, DAG);
8075 if (
LHS == LHS2 &&
RHS == RHS2) {
8080 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
8088 return std::nullopt;
8096 MVT VT =
N->getSimpleValueType(0);
8126 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
8129 if (~TrueVal == FalseVal) {
8169 if (Subtarget.hasShortForwardBranchOpt())
8172 unsigned SelOpNo = 0;
8182 unsigned ConstSelOpNo = 1;
8183 unsigned OtherSelOpNo = 2;
8184 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
8189 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
8190 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
8194 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
8195 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
8201 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
8203 std::swap(NewConstOps[0], NewConstOps[1]);
8215 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
8217 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
8220 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
8221 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
8230 MVT VT =
Op.getSimpleValueType();
8244 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
8272 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
8276 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
8278 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
8279 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
8281 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
8286 DL, VT, LHSVal, CondV);
8302 if (
Op.hasOneUse()) {
8303 unsigned UseOpc =
Op->user_begin()->getOpcode();
8312 return lowerSELECT(NewSel, DAG);
8341 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
8362 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
8366 if (TrueVal - 1 == FalseVal)
8368 if (TrueVal + 1 == FalseVal)
8375 RHS == TrueV && LHS == FalseV) {
8392 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
8418 LHS, RHS, TargetCC,
Op.getOperand(2));
8436 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
8448 int XLenInBytes = Subtarget.
getXLen() / 8;
8450 EVT VT =
Op.getValueType();
8453 unsigned Depth =
Op.getConstantOperandVal(0);
8455 int Offset = -(XLenInBytes * 2);
8472 int XLenInBytes = Subtarget.
getXLen() / 8;
8477 EVT VT =
Op.getValueType();
8479 unsigned Depth =
Op.getConstantOperandVal(0);
8481 int Off = -XLenInBytes;
8482 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
8501 EVT VT =
Lo.getValueType();
8540 EVT VT =
Lo.getValueType();
8591 MVT VT =
Op.getSimpleValueType();
8617 MVT VecVT =
Op.getSimpleValueType();
8619 "Unexpected SPLAT_VECTOR_PARTS lowering");
8625 MVT ContainerVT = VecVT;
8645 int64_t ExtTrueVal)
const {
8647 MVT VecVT =
Op.getSimpleValueType();
8650 assert(Src.getValueType().isVector() &&
8651 Src.getValueType().getVectorElementType() == MVT::i1);
8672 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8674 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
8677 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
8682SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
8684 MVT ExtVT =
Op.getSimpleValueType();
8688 MVT VT =
Op.getOperand(0).getSimpleValueType();
8714 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8716 EVT MaskVT =
Op.getValueType();
8719 "Unexpected type for vector mask lowering");
8721 MVT VecVT = Src.getSimpleValueType();
8725 VL =
Op.getOperand(2);
8728 MVT ContainerVT = VecVT;
8734 MVT MaskContainerVT =
8741 std::tie(Mask, VL) =
8749 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8751 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8755 DAG.
getUNDEF(ContainerVT), Mask, VL);
8758 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8766 unsigned Opc =
Op.getOpcode();
8767 bool IsVPTrunc = Opc == ISD::VP_TRUNCATE;
8770 MVT VT =
Op.getSimpleValueType();
8772 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8776 return lowerVectorMaskTruncLike(
Op, DAG);
8784 MVT SrcVT = Src.getSimpleValueType();
8789 "Unexpected vector truncate lowering");
8791 MVT ContainerVT = SrcVT;
8795 VL =
Op.getOperand(2);
8808 std::tie(Mask, VL) =
8824 }
while (SrcEltVT != DstEltVT);
8833RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8838 MVT VT =
Op.getSimpleValueType();
8839 MVT SrcVT = Src.getSimpleValueType();
8840 MVT ContainerVT = VT;
8863 Chain, Src, Mask, VL);
8864 Chain = Src.getValue(1);
8871 Chain, Src, Mask, VL);
8882RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8885 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8892 MVT VT =
Op.getSimpleValueType();
8894 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8897 MVT SrcVT = Src.getSimpleValueType();
8899 bool IsDirectExtend =
8907 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8910 MVT ContainerVT = VT;
8914 VL =
Op.getOperand(2);
8928 std::tie(Mask, VL) =
8934 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8940 unsigned InterConvOpc =
8945 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8947 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8958static std::optional<MVT>
8964 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8966 if (MaxIdx < MinVLMAX)
8968 else if (MaxIdx < MinVLMAX * 2)
8970 else if (MaxIdx < MinVLMAX * 4)
8975 return std::nullopt;
8988 MVT VecVT =
Op.getSimpleValueType();
9005 ValVT == MVT::bf16) {
9014 MVT ContainerVT = VecVT;
9023 MVT OrigContainerVT = ContainerVT;
9026 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
9027 const unsigned OrigIdx = IdxC->getZExtValue();
9030 DL, DAG, Subtarget)) {
9031 ContainerVT = *ShrunkVT;
9040 VLEN && ContainerVT.
bitsGT(M1VT)) {
9043 unsigned RemIdx = OrigIdx % ElemsPerVReg;
9044 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
9045 unsigned ExtractIdx =
9062 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
9063 const auto *CVal = cast<ConstantSDNode>(Val);
9064 if (isInt<32>(CVal->getSExtValue())) {
9065 IsLegalInsert =
true;
9074 if (IsLegalInsert) {
9080 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
9096 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
9097 MVT I32ContainerVT =
9108 Vec, Vec, ValLo, I32Mask, InsertI64VL);
9113 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
9115 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
9120 ValInVec, AlignedIdx);
9130 DAG.
getUNDEF(I32ContainerVT), ValLo,
9131 I32Mask, InsertI64VL);
9133 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
9134 I32Mask, InsertI64VL);
9136 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
9149 Idx, Mask, InsertVL, Policy);
9153 Slideup, AlignedIdx);
9168 EVT EltVT =
Op.getValueType();
9175 MVT ContainerVT = VecVT;
9191 unsigned WidenVecLen;
9194 unsigned MaxEEW = Subtarget.
getELen();
9199 "the number of elements should be power of 2");
9203 ExtractBitIdx =
Idx;
9205 WideEltVT = LargestEltVT;
9208 ExtractElementIdx = DAG.
getNode(
9219 Vec, ExtractElementIdx);
9235 EltVT == MVT::bf16) {
9245 MVT ContainerVT = VecVT;
9256 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
9257 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
9259 unsigned OrigIdx = IdxC->getZExtValue();
9262 unsigned RemIdx = OrigIdx % ElemsPerVReg;
9263 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
9264 unsigned ExtractIdx =
9274 std::optional<uint64_t> MaxIdx;
9277 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
9278 MaxIdx = IdxC->getZExtValue();
9280 if (
auto SmallerVT =
9282 ContainerVT = *SmallerVT;
9329 "Unexpected opcode");
9336 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9341 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9342 if (!
II || !
II->hasScalarOperand())
9345 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
9358 if (OpVT.
bitsLT(XLenVT)) {
9365 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9375 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
9376 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
9379 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
9390 case Intrinsic::riscv_vslide1up:
9391 case Intrinsic::riscv_vslide1down:
9392 case Intrinsic::riscv_vslide1up_mask:
9393 case Intrinsic::riscv_vslide1down_mask: {
9396 bool IsMasked = NumOps == 7;
9402 std::tie(ScalarLo, ScalarHi) =
9410 if (isa<ConstantSDNode>(AVL)) {
9411 const auto [MinVLMAX, MaxVLMAX] =
9415 if (AVLInt <= MinVLMAX) {
9417 }
else if (AVLInt >= 2 * MaxVLMAX) {
9451 if (IntNo == Intrinsic::riscv_vslide1up ||
9452 IntNo == Intrinsic::riscv_vslide1up_mask) {
9454 ScalarHi, I32Mask, I32VL);
9456 ScalarLo, I32Mask, I32VL);
9459 ScalarLo, I32Mask, I32VL);
9461 ScalarHi, I32Mask, I32VL);
9510 const unsigned ElementWidth = 8;
9515 [[maybe_unused]]
unsigned MinVF =
9518 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
9522 bool Fractional = VF < LMul1VF;
9523 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
9544 MVT ContainerVT = OpVT;
9571 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9575 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9576 if (!
II || !
II->hasScalarOperand())
9579 unsigned SplatOp =
II->ScalarOperand + 1;
9592 if (OpVT.
bitsLT(XLenVT)) {
9595 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9606 EVT ValType = V.getValueType();
9607 if (ValType.isVector() && ValType.isFloatingPoint()) {
9610 ValType.getVectorElementCount());
9613 if (ValType.isFixedLengthVector()) {
9615 DAG, V.getSimpleValueType(), Subtarget);
9631 unsigned IntNo =
Op.getConstantOperandVal(0);
9638 case Intrinsic::riscv_tuple_insert: {
9646 case Intrinsic::riscv_tuple_extract: {
9653 case Intrinsic::thread_pointer: {
9657 case Intrinsic::riscv_orc_b:
9658 case Intrinsic::riscv_brev8:
9659 case Intrinsic::riscv_sha256sig0:
9660 case Intrinsic::riscv_sha256sig1:
9661 case Intrinsic::riscv_sha256sum0:
9662 case Intrinsic::riscv_sha256sum1:
9663 case Intrinsic::riscv_sm3p0:
9664 case Intrinsic::riscv_sm3p1: {
9677 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9679 case Intrinsic::riscv_sm4ks:
9680 case Intrinsic::riscv_sm4ed: {
9684 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
9687 case Intrinsic::riscv_zip:
9688 case Intrinsic::riscv_unzip: {
9691 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9693 case Intrinsic::riscv_mopr:
9697 case Intrinsic::riscv_moprr: {
9699 Op.getOperand(2),
Op.getOperand(3));
9701 case Intrinsic::riscv_clmul:
9704 case Intrinsic::riscv_clmulh:
9705 case Intrinsic::riscv_clmulr: {
9708 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9710 case Intrinsic::experimental_get_vector_length:
9712 case Intrinsic::experimental_cttz_elts:
9714 case Intrinsic::riscv_vmv_x_s: {
9718 case Intrinsic::riscv_vfmv_f_s:
9721 case Intrinsic::riscv_vmv_v_x:
9723 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9725 case Intrinsic::riscv_vfmv_v_f:
9727 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9728 case Intrinsic::riscv_vmv_s_x: {
9731 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9734 Op.getOperand(1), Scalar,
Op.getOperand(3));
9737 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9754 MVT VT =
Op.getSimpleValueType();
9759 if (
Op.getOperand(1).isUndef())
9775 case Intrinsic::riscv_vfmv_s_f:
9777 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9779 case Intrinsic::riscv_vaesdf_vv:
9780 case Intrinsic::riscv_vaesdf_vs:
9781 case Intrinsic::riscv_vaesdm_vv:
9782 case Intrinsic::riscv_vaesdm_vs:
9783 case Intrinsic::riscv_vaesef_vv:
9784 case Intrinsic::riscv_vaesef_vs:
9785 case Intrinsic::riscv_vaesem_vv:
9786 case Intrinsic::riscv_vaesem_vs:
9787 case Intrinsic::riscv_vaeskf1:
9788 case Intrinsic::riscv_vaeskf2:
9789 case Intrinsic::riscv_vaesz_vs:
9790 case Intrinsic::riscv_vsm4k:
9791 case Intrinsic::riscv_vsm4r_vv:
9792 case Intrinsic::riscv_vsm4r_vs: {
9793 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9794 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9795 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9800 case Intrinsic::riscv_vsm3c:
9801 case Intrinsic::riscv_vsm3me: {
9802 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9803 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9808 case Intrinsic::riscv_vsha2ch:
9809 case Intrinsic::riscv_vsha2cl:
9810 case Intrinsic::riscv_vsha2ms: {
9811 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9812 !Subtarget.hasStdExtZvknhb())
9814 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9815 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9816 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9820 case Intrinsic::riscv_sf_vc_v_x:
9821 case Intrinsic::riscv_sf_vc_v_i:
9822 case Intrinsic::riscv_sf_vc_v_xv:
9823 case Intrinsic::riscv_sf_vc_v_iv:
9824 case Intrinsic::riscv_sf_vc_v_vv:
9825 case Intrinsic::riscv_sf_vc_v_fv:
9826 case Intrinsic::riscv_sf_vc_v_xvv:
9827 case Intrinsic::riscv_sf_vc_v_ivv:
9828 case Intrinsic::riscv_sf_vc_v_vvv:
9829 case Intrinsic::riscv_sf_vc_v_fvv:
9830 case Intrinsic::riscv_sf_vc_v_xvw:
9831 case Intrinsic::riscv_sf_vc_v_ivw:
9832 case Intrinsic::riscv_sf_vc_v_vvw:
9833 case Intrinsic::riscv_sf_vc_v_fvw: {
9834 MVT VT =
Op.getSimpleValueType();
9871 MVT VT =
Op.getSimpleValueType();
9875 if (VT.isFloatingPoint()) {
9880 if (VT.isFixedLengthVector())
9890 if (VT.isFixedLengthVector())
9892 if (VT.isFloatingPoint())
9911 unsigned IntNo =
Op.getConstantOperandVal(1);
9915 case Intrinsic::riscv_seg2_load:
9916 case Intrinsic::riscv_seg3_load:
9917 case Intrinsic::riscv_seg4_load:
9918 case Intrinsic::riscv_seg5_load:
9919 case Intrinsic::riscv_seg6_load:
9920 case Intrinsic::riscv_seg7_load:
9921 case Intrinsic::riscv_seg8_load: {
9924 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9925 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9926 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9927 Intrinsic::riscv_vlseg8};
9928 unsigned NF =
Op->getNumValues() - 1;
9929 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9931 MVT VT =
Op->getSimpleValueType(0);
9939 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9951 Load->getMemoryVT(),
Load->getMemOperand());
9953 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
9962 case Intrinsic::riscv_sf_vc_v_x_se:
9964 case Intrinsic::riscv_sf_vc_v_i_se:
9966 case Intrinsic::riscv_sf_vc_v_xv_se:
9968 case Intrinsic::riscv_sf_vc_v_iv_se:
9970 case Intrinsic::riscv_sf_vc_v_vv_se:
9972 case Intrinsic::riscv_sf_vc_v_fv_se:
9974 case Intrinsic::riscv_sf_vc_v_xvv_se:
9976 case Intrinsic::riscv_sf_vc_v_ivv_se:
9978 case Intrinsic::riscv_sf_vc_v_vvv_se:
9980 case Intrinsic::riscv_sf_vc_v_fvv_se:
9982 case Intrinsic::riscv_sf_vc_v_xvw_se:
9984 case Intrinsic::riscv_sf_vc_v_ivw_se:
9986 case Intrinsic::riscv_sf_vc_v_vvw_se:
9988 case Intrinsic::riscv_sf_vc_v_fvw_se:
9997 unsigned IntNo =
Op.getConstantOperandVal(1);
10001 case Intrinsic::riscv_seg2_store:
10002 case Intrinsic::riscv_seg3_store:
10003 case Intrinsic::riscv_seg4_store:
10004 case Intrinsic::riscv_seg5_store:
10005 case Intrinsic::riscv_seg6_store:
10006 case Intrinsic::riscv_seg7_store:
10007 case Intrinsic::riscv_seg8_store: {
10010 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
10011 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
10012 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
10013 Intrinsic::riscv_vsseg8};
10016 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
10018 MVT VT =
Op->getOperand(2).getSimpleValueType();
10028 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
10031 for (
unsigned i = 0; i < NF; i++)
10035 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget),
10039 FixedIntrinsic->getChain(),
10048 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
10050 case Intrinsic::riscv_sf_vc_xv_se:
10052 case Intrinsic::riscv_sf_vc_iv_se:
10054 case Intrinsic::riscv_sf_vc_vv_se:
10056 case Intrinsic::riscv_sf_vc_fv_se:
10058 case Intrinsic::riscv_sf_vc_xvv_se:
10060 case Intrinsic::riscv_sf_vc_ivv_se:
10062 case Intrinsic::riscv_sf_vc_vvv_se:
10064 case Intrinsic::riscv_sf_vc_fvv_se:
10066 case Intrinsic::riscv_sf_vc_xvw_se:
10068 case Intrinsic::riscv_sf_vc_ivw_se:
10070 case Intrinsic::riscv_sf_vc_vvw_se:
10072 case Intrinsic::riscv_sf_vc_fvw_se:
10080 switch (ISDOpcode) {
10083 case ISD::VP_REDUCE_ADD:
10086 case ISD::VP_REDUCE_UMAX:
10089 case ISD::VP_REDUCE_SMAX:
10092 case ISD::VP_REDUCE_UMIN:
10095 case ISD::VP_REDUCE_SMIN:
10098 case ISD::VP_REDUCE_AND:
10101 case ISD::VP_REDUCE_OR:
10104 case ISD::VP_REDUCE_XOR:
10107 case ISD::VP_REDUCE_FADD:
10109 case ISD::VP_REDUCE_SEQ_FADD:
10111 case ISD::VP_REDUCE_FMAX:
10112 case ISD::VP_REDUCE_FMAXIMUM:
10114 case ISD::VP_REDUCE_FMIN:
10115 case ISD::VP_REDUCE_FMINIMUM:
10125 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
10130 Op.getOpcode() == ISD::VP_REDUCE_AND ||
10131 Op.getOpcode() == ISD::VP_REDUCE_OR ||
10132 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
10133 "Unexpected reduction lowering");
10137 MVT ContainerVT = VecVT;
10145 Mask =
Op.getOperand(2);
10146 VL =
Op.getOperand(3);
10148 std::tie(Mask, VL) =
10153 switch (
Op.getOpcode()) {
10157 case ISD::VP_REDUCE_AND: {
10166 case ISD::VP_REDUCE_OR:
10172 case ISD::VP_REDUCE_XOR: {
10196 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
10200 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
10201 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
10202 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
10203 (ImmAVL && ImmAVL->getZExtValue() >= 1);
10219 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
10223 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
10226 if (M1VT != InnerVT)
10232 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
10251 VecEVT =
Lo.getValueType();
10264 MVT ContainerVT = VecVT;
10284 Mask, VL,
DL, DAG, Subtarget);
10290static std::tuple<unsigned, SDValue, SDValue>
10294 auto Flags =
Op->getFlags();
10295 unsigned Opcode =
Op.getOpcode();
10319 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
10327 MVT VecEltVT =
Op.getSimpleValueType();
10329 unsigned RVVOpcode;
10330 SDValue VectorVal, ScalarVal;
10331 std::tie(RVVOpcode, VectorVal, ScalarVal) =
10335 MVT ContainerVT = VecVT;
10341 MVT ResVT =
Op.getSimpleValueType();
10344 VL,
DL, DAG, Subtarget);
10349 if (
Op->getFlags().hasNoNaNs())
10355 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
10356 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
10362 DL, ResVT, NoNaNs, Res,
10369 unsigned Opc =
Op.getOpcode();
10392 Vec, Mask, VL,
DL, DAG, Subtarget);
10393 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
10394 Op->getFlags().hasNoNaNs())
10411 DL, ResVT, NoNaNs, Res,
10424 unsigned OrigIdx =
Op.getConstantOperandVal(2);
10427 if (OrigIdx == 0 && Vec.
isUndef())
10438 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10441 "Unexpected mask vector lowering");
10473 MVT ContainerVT = VecVT;
10480 DAG.
getUNDEF(ContainerVT), SubVec,
10497 if (OrigIdx == 0) {
10502 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
10503 SlideupAmt, Mask, VL, Policy);
10511 MVT ContainerVecVT = VecVT;
10517 MVT ContainerSubVecVT = SubVecVT;
10523 unsigned SubRegIdx;
10533 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10534 SubRegIdx = Decompose.first;
10536 (OrigIdx % Vscale));
10540 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
10541 SubRegIdx = Decompose.first;
10548 bool ExactlyVecRegSized =
10550 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
10565 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10569 if (SubRegIdx == RISCV::NoSubRegister) {
10592 MVT InterSubVT = ContainerVecVT;
10593 SDValue AlignedExtract = Vec;
10633 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10634 SlideupAmt, Mask, VL, Policy);
10639 if (ContainerVecVT.
bitsGT(InterSubVT))
10648 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10654 MVT SubVecVT =
Op.getSimpleValueType();
10659 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10675 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10678 "Unexpected mask vector lowering");
10712 MVT ContainerVT = VecVT;
10720 if (
auto ShrunkVT =
10722 ContainerVT = *ShrunkVT;
10735 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10747 MVT ContainerSubVecVT = SubVecVT;
10751 unsigned SubRegIdx;
10761 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10762 SubRegIdx = Decompose.first;
10764 (OrigIdx % Vscale));
10768 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10769 SubRegIdx = Decompose.first;
10796 MVT InterSubVT = VecVT;
10801 assert(SubRegIdx != RISCV::NoSubRegister);
10821 Vec, SlidedownAmt, Mask, VL);
10830 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10837 MVT VT =
N.getSimpleValueType();
10841 assert(
Op.getSimpleValueType() == VT &&
10842 "Operands and result must be same type");
10846 unsigned NumVals =
N->getNumValues();
10849 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10852 for (
unsigned I = 0;
I < NumVals;
I++) {
10858 if (TruncVals.
size() > 1)
10860 return TruncVals.
front();
10866 MVT VecVT =
Op.getSimpleValueType();
10869 "vector_interleave on non-scalable vector!");
10880 EVT SplitVT = Op0Lo.getValueType();
10883 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10885 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10899 Op.getOperand(0),
Op.getOperand(1));
10918 EvenSplat = DAG.
getBitcast(MVT::nxv64i1, EvenSplat);
10923 OddSplat = DAG.
getBitcast(MVT::nxv64i1, OddSplat);
10929 EvenMask, DAG.
getUNDEF(ConcatVT));
10945 MVT VecVT =
Op.getSimpleValueType();
10948 "vector_interleave on non-scalable vector!");
10961 EVT SplitVT = Op0Lo.getValueType();
10964 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10966 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10988 Op.getOperand(0),
Op.getOperand(1));
11036 MVT VT =
Op.getSimpleValueType();
11041 uint64_t StepValImm =
Op.getConstantOperandVal(0);
11042 if (StepValImm != 1) {
11051 VL, VT,
DL, DAG, Subtarget);
11066 MVT VecVT =
Op.getSimpleValueType();
11075 MVT ContainerVT = VecVT;
11129 unsigned MaxVLMAX =
11139 if (MaxVLMAX > 256 && EltSize == 8) {
11168 assert(isUInt<16>(MaxVLMAX - 1));
11194 DAG.
getUNDEF(ContainerVT), Mask, VL);
11206 MVT VecVT =
Op.getSimpleValueType();
11210 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
11211 SDValue DownOffset, UpOffset;
11212 if (ImmValue >= 0) {
11228 DownOffset, TrueMask, UpOffset);
11229 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
11235RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
11238 auto *
Load = cast<LoadSDNode>(
Op);
11241 Load->getMemoryVT(),
11242 *
Load->getMemOperand()) &&
11243 "Expecting a correctly-aligned load");
11245 MVT VT =
Op.getSimpleValueType();
11251 const auto [MinVLMAX, MaxVLMAX] =
11254 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
11268 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
11271 Ops.push_back(DAG.
getUNDEF(ContainerVT));
11272 Ops.push_back(
Load->getBasePtr());
11277 Load->getMemoryVT(),
Load->getMemOperand());
11284RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
11287 auto *
Store = cast<StoreSDNode>(
Op);
11290 Store->getMemoryVT(),
11291 *
Store->getMemOperand()) &&
11292 "Expecting a correctly-aligned store");
11313 const auto [MinVLMAX, MaxVLMAX] =
11316 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
11327 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
11330 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
11331 Store->getMemoryVT(),
Store->getMemOperand());
11337 MVT VT =
Op.getSimpleValueType();
11339 const auto *MemSD = cast<MemSDNode>(
Op);
11340 EVT MemVT = MemSD->getMemoryVT();
11342 SDValue Chain = MemSD->getChain();
11346 bool IsExpandingLoad =
false;
11347 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
11348 Mask = VPLoad->getMask();
11350 VL = VPLoad->getVectorLength();
11352 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
11353 Mask = MLoad->getMask();
11354 PassThru = MLoad->getPassThru();
11355 IsExpandingLoad = MLoad->isExpandingLoad();
11362 MVT ContainerVT = VT;
11376 if (!IsUnmasked && IsExpandingLoad) {
11383 unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
11384 : Intrinsic::riscv_vle_mask;
11386 if (IntID == Intrinsic::riscv_vle)
11387 Ops.push_back(DAG.
getUNDEF(ContainerVT));
11389 Ops.push_back(PassThru);
11390 Ops.push_back(BasePtr);
11391 if (IntID == Intrinsic::riscv_vle_mask)
11392 Ops.push_back(Mask);
11394 if (IntID == Intrinsic::riscv_vle_mask)
11401 Chain =
Result.getValue(1);
11403 MVT IndexVT = ContainerVT;
11408 bool UseVRGATHEREI16 =
false;
11416 UseVRGATHEREI16 =
true;
11422 DAG.
getUNDEF(IndexVT), Mask, ExpandingVL);
11426 DL, ContainerVT, Result, Iota, PassThru, Mask, ExpandingVL);
11439 const auto *MemSD = cast<MemSDNode>(
Op);
11440 EVT MemVT = MemSD->getMemoryVT();
11442 SDValue Chain = MemSD->getChain();
11446 bool IsCompressingStore =
false;
11447 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
11448 Val = VPStore->getValue();
11449 Mask = VPStore->getMask();
11450 VL = VPStore->getVectorLength();
11452 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
11453 Val = MStore->getValue();
11454 Mask = MStore->getMask();
11455 IsCompressingStore = MStore->isCompressingStore();
11464 MVT ContainerVT = VT;
11469 if (!IsUnmasked || IsCompressingStore) {
11478 if (IsCompressingStore) {
11481 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
11488 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
11490 Ops.push_back(Val);
11491 Ops.push_back(BasePtr);
11493 Ops.push_back(Mask);
11497 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11509 MVT ContainerVT = VT;
11522 Passthru, Val, Mask, VL);
11531RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
11533 MVT InVT =
Op.getOperand(0).getSimpleValueType();
11536 MVT VT =
Op.getSimpleValueType();
11550 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
11557 unsigned Opc =
Op.getOpcode();
11564 MVT VT =
Op.getSimpleValueType();
11597 MVT ContainerInVT = InVT;
11616 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11620 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11628 {Chain, Op1, Op2, CC, Mask, Mask, VL});
11633 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
11646 MVT VT =
Op.getSimpleValueType();
11650 "Unexpected type for ISD::ABS");
11652 MVT ContainerVT = VT;
11659 if (
Op->getOpcode() == ISD::VP_ABS) {
11660 Mask =
Op->getOperand(1);
11664 VL =
Op->getOperand(2);
11672 DAG.
getUNDEF(ContainerVT), Mask, VL);
11674 DAG.
getUNDEF(ContainerVT), Mask, VL);
11681SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
11684 MVT VT =
Op.getSimpleValueType();
11688 "Can only handle COPYSIGN with matching types.");
11697 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
11702SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
11704 MVT VT =
Op.getSimpleValueType();
11707 MVT I1ContainerVT =
11721 Op2, DAG.
getUNDEF(ContainerVT), VL);
11732 MVT VT =
Op.getSimpleValueType();
11737 for (
const SDValue &V :
Op->op_values()) {
11738 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11741 if (!
V.getValueType().isVector()) {
11747 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
11748 "Only fixed length vectors are supported!");
11762 if (
Op->isStrictFPOpcode()) {
11771 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11785 MVT VT =
Op.getSimpleValueType();
11788 MVT ContainerVT = VT;
11794 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11797 if (HasPassthruOp) {
11800 if (*MaskIdx == OpIdx.index())
11804 if (
Op.getOpcode() == ISD::VP_MERGE) {
11808 assert(
Op.getOpcode() == ISD::VP_SELECT);
11820 if (!
V.getValueType().isFixedLengthVector()) {
11825 MVT OpVT =
V.getSimpleValueType();
11827 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11828 "Only fixed length vectors are supported!");
11833 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11843 MVT VT =
Op.getSimpleValueType();
11849 MVT ContainerVT = VT;
11859 DAG.
getUNDEF(ContainerVT), Zero, VL);
11862 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11864 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11867 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11876 MVT VT =
Op.getSimpleValueType();
11880 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11884 MVT ContainerVT = VT;
11894 switch (Condition) {
11962 MVT DstVT =
Op.getSimpleValueType();
11963 MVT SrcVT = Src.getSimpleValueType();
11976 if (DstEltSize >= SrcEltSize) {
11985 if (SrcEltSize == 1) {
11996 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11997 }
else if (DstEltSize > (2 * SrcEltSize)) {
12001 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
12007 "Wrong input/output vector types");
12010 if (DstEltSize > (2 * SrcEltSize)) {
12026 MVT InterimFVT = DstVT;
12027 if (SrcEltSize > (2 * DstEltSize)) {
12028 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
12035 if (InterimFVT != DstVT) {
12041 "Wrong input/output vector types");
12045 if (DstEltSize == 1) {
12048 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
12058 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
12068 while (InterimIVT != DstVT) {
12080 MVT VT =
Op.getSimpleValueType();
12089 MVT VT =
Op.getSimpleValueType();
12103 MVT ContainerVT = VT;
12124 SplatZero, DAG.
getUNDEF(PromotedVT), VL);
12127 SplatOne, SplatZero, DAG.
getUNDEF(PromotedVT), VLMax);
12131 TrueVal, FalseVal, FalseVal, VL);
12146RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
12158 MVT VT =
Op.getSimpleValueType();
12159 MVT ContainerVT = VT;
12169 if (IsMaskVector) {
12180 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
12189 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
12192 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
12193 SDValue DownOffset, UpOffset;
12194 if (ImmValue >= 0) {
12208 Op1, DownOffset, Mask, UpOffset);
12212 if (IsMaskVector) {
12216 {Result, DAG.getConstant(0, DL, ContainerVT),
12217 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
12232 MVT VT =
Op.getSimpleValueType();
12234 MVT ContainerVT = VT;
12250RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
12253 MVT VT =
Op.getSimpleValueType();
12260 MVT ContainerVT = VT;
12268 MVT GatherVT = ContainerVT;
12272 if (IsMaskVector) {
12283 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
12289 unsigned MaxVLMAX =
12298 if (MaxVLMAX > 256 && EltSize == 8) {
12326 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
12328 if (IsMaskVector) {
12351 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
12353 DAG.
getUNDEF(IndicesVT), Mask, EVL);
12355 DAG.
getUNDEF(GatherVT), Mask, EVL);
12357 if (IsMaskVector) {
12372 MVT VT =
Op.getSimpleValueType();
12374 return lowerVPOp(
Op, DAG);
12381 MVT ContainerVT = VT;
12400 MVT VT =
Op.getSimpleValueType();
12401 MVT ContainerVT = VT;
12407 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
12413 : Intrinsic::riscv_vlse_mask,
12416 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
12417 VPNode->getStride()};
12425 Ops.
push_back(VPNode->getVectorLength());
12433 VPNode->getMemoryVT(), VPNode->getMemOperand());
12447 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
12448 SDValue StoreVal = VPNode->getValue();
12450 MVT ContainerVT = VT;
12461 : Intrinsic::riscv_vsse_mask,
12464 VPNode->getBasePtr(), VPNode->getStride()};
12472 Ops.
push_back(VPNode->getVectorLength());
12475 Ops, VPNode->getMemoryVT(),
12476 VPNode->getMemOperand());
12488 MVT VT =
Op.getSimpleValueType();
12490 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
12491 EVT MemVT = MemSD->getMemoryVT();
12493 SDValue Chain = MemSD->getChain();
12499 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
12500 Index = VPGN->getIndex();
12501 Mask = VPGN->getMask();
12503 VL = VPGN->getVectorLength();
12508 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
12509 Index = MGN->getIndex();
12510 Mask = MGN->getMask();
12511 PassThru = MGN->getPassThru();
12515 MVT IndexVT =
Index.getSimpleValueType();
12519 "Unexpected VTs!");
12520 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12523 "Unexpected extending MGATHER/VP_GATHER");
12529 MVT ContainerVT = VT;
12553 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
12570 Chain =
Result.getValue(1);
12587 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
12588 EVT MemVT = MemSD->getMemoryVT();
12590 SDValue Chain = MemSD->getChain();
12593 [[maybe_unused]]
bool IsTruncatingStore =
false;
12596 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
12597 Index = VPSN->getIndex();
12598 Mask = VPSN->getMask();
12599 Val = VPSN->getValue();
12600 VL = VPSN->getVectorLength();
12602 IsTruncatingStore =
false;
12605 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
12606 Index = MSN->getIndex();
12607 Mask = MSN->getMask();
12608 Val = MSN->getValue();
12609 IsTruncatingStore = MSN->isTruncatingStore();
12613 MVT IndexVT =
Index.getSimpleValueType();
12617 "Unexpected VTs!");
12618 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
12621 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
12627 MVT ContainerVT = VT;
12651 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
12661 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
12670 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
12678 static const int Table =
12702 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
12708 static const unsigned Table =
12731 bool isRISCV64 = Subtarget.
is64Bit();
12795 switch (
N->getOpcode()) {
12797 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12803 "Unexpected custom legalisation");
12804 bool IsStrict =
N->isStrictFPOpcode();
12807 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12825 Opc,
DL, VTs, Chain, Op0,
12859 std::tie(Result, Chain) =
12860 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12888 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12899 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12900 "has custom type legalization on riscv32");
12902 SDValue LoCounter, HiCounter;
12906 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12908 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12911 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12913 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12917 N->getOperand(0), LoCounter, HiCounter);
12941 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12942 unsigned XLen = Subtarget.
getXLen();
12945 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12953 if (LHSIsU == RHSIsU)
12970 if (RHSIsU && LHSIsS && !RHSIsS)
12972 else if (LHSIsU && RHSIsS && !LHSIsS)
12982 "Unexpected custom legalisation");
12989 "Unexpected custom legalisation");
12992 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
13018 "Unexpected custom legalisation");
13019 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
13020 Subtarget.hasVendorXTHeadBb()) &&
13021 "Unexpected custom legalization");
13022 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
13023 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
13032 "Unexpected custom legalisation");
13046 MVT VT =
N->getSimpleValueType(0);
13047 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
13048 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
13049 "Unexpected custom legalisation");
13061 if (VT != MVT::i32)
13070 "Unexpected custom legalisation");
13074 if (!isa<ConstantSDNode>(
N->getOperand(1)))
13091 EVT OType =
N->getValueType(1);
13104 "Unexpected custom legalisation");
13121 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
13125 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
13143 !Subtarget.hasStdExtZbb() &&
"Unexpected custom legalisation");
13152 "Unexpected custom legalisation");
13158 "Unexpected custom legalisation");
13160 if (Subtarget.hasStdExtZbb()) {
13194 EVT VT =
N->getValueType(0);
13199 if (VT == MVT::i16 &&
13201 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
13204 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
13209 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.
is64Bit() &&
13212 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
13233 MVT VT =
N->getSimpleValueType(0);
13235 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
13236 "Unexpected custom legalisation");
13239 "Unexpected extension");
13265 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
13267 "Unexpected EXTRACT_VECTOR_ELT legalization");
13270 MVT ContainerVT = VecVT;
13298 DAG.
getUNDEF(ContainerVT), Mask, VL);
13306 unsigned IntNo =
N->getConstantOperandVal(0);
13310 "Don't know how to custom type legalize this intrinsic!");
13311 case Intrinsic::experimental_get_vector_length: {
13316 case Intrinsic::experimental_cttz_elts: {
13322 case Intrinsic::riscv_orc_b:
13323 case Intrinsic::riscv_brev8:
13324 case Intrinsic::riscv_sha256sig0:
13325 case Intrinsic::riscv_sha256sig1:
13326 case Intrinsic::riscv_sha256sum0:
13327 case Intrinsic::riscv_sha256sum1:
13328 case Intrinsic::riscv_sm3p0:
13329 case Intrinsic::riscv_sm3p1: {
13330 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13350 case Intrinsic::riscv_sm4ks:
13351 case Intrinsic::riscv_sm4ed: {
13359 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
13363 case Intrinsic::riscv_mopr: {
13364 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13374 case Intrinsic::riscv_moprr: {
13375 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13387 case Intrinsic::riscv_clmul: {
13388 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13399 case Intrinsic::riscv_clmulh:
13400 case Intrinsic::riscv_clmulr: {
13401 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
13429 case Intrinsic::riscv_vmv_x_s: {
13430 EVT VT =
N->getValueType(0);
13432 if (VT.
bitsLT(XLenVT)) {
13441 "Unexpected custom legalization");
13479 case ISD::VP_REDUCE_ADD:
13480 case ISD::VP_REDUCE_AND:
13481 case ISD::VP_REDUCE_OR:
13482 case ISD::VP_REDUCE_XOR:
13483 case ISD::VP_REDUCE_SMAX:
13484 case ISD::VP_REDUCE_UMAX:
13485 case ISD::VP_REDUCE_SMIN:
13486 case ISD::VP_REDUCE_UMIN:
13550 const EVT VT =
N->getValueType(0);
13551 const unsigned Opc =
N->getOpcode();
13558 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
13563 "Inconsistent mappings");
13574 !isa<ConstantSDNode>(
RHS.getOperand(1)))
13577 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
13592 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
13594 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
13595 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
13599 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
13606 if (
LHS.getOpcode() != ReduceOpc)
13621 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
13622 ReduceVec->
getFlags() &
N->getFlags());
13632 auto BinOpToRVVReduce = [](
unsigned Opc) {
13661 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
13664 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
13667 unsigned Opc =
N->getOpcode();
13668 unsigned ReduceIdx;
13669 if (IsReduction(
N->getOperand(0), Opc))
13671 else if (IsReduction(
N->getOperand(1), Opc))
13677 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
13680 SDValue Extract =
N->getOperand(ReduceIdx);
13712 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
13739 if (!Subtarget.hasStdExtZba())
13743 EVT VT =
N->getValueType(0);
13755 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13756 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
13759 int64_t C0 = N0C->getSExtValue();
13760 int64_t C1 = N1C->getSExtValue();
13761 if (C0 <= 0 || C1 <= 0)
13765 int64_t Bits = std::min(C0, C1);
13766 int64_t Diff = std::abs(C0 - C1);
13767 if (Diff != 1 && Diff != 2 && Diff != 3)
13794 EVT VT =
N->getValueType(0);
13802 if ((!Subtarget.hasStdExtZicond() &&
13803 !Subtarget.hasVendorXVentanaCondOps()) ||
13825 bool SwapSelectOps;
13831 SwapSelectOps =
false;
13832 NonConstantVal = FalseVal;
13834 SwapSelectOps =
true;
13835 NonConstantVal = TrueVal;
13841 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13889 EVT VT =
N->getValueType(0);
13897 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13898 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13904 if (!N0C->hasOneUse())
13906 int64_t C0 = N0C->getSExtValue();
13907 int64_t C1 = N1C->getSExtValue();
13909 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13912 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
13913 !isInt<12>(C0 * (C1 / C0))) {
13916 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
13917 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13920 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13921 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13948 EVT VT =
N->getValueType(0);
13979 unsigned OuterExtend =
13983 OuterExtend,
SDLoc(
N), VT,
13991 EVT VT =
N->getValueType(0);
14039 EVT VT =
N->getValueType(0);
14043 auto *N0C = dyn_cast<ConstantSDNode>(N0);
14049 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
14059 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
14082 if (!Subtarget.hasStdExtZbb())
14085 EVT VT =
N->getValueType(0);
14087 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
14096 auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
14099 unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
14101 if (ShiftedAmount >= 8)
14105 SDValue RightShiftOperand = N1;
14107 if (ShiftedAmount != 0) {
14110 auto *ShAmtCRight = dyn_cast<ConstantSDNode>(N1.
getOperand(1));
14111 if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
14120 if (LeftShiftOperand != RightShiftOperand)
14124 Mask <<= ShiftedAmount;
14138 EVT VT =
N->getValueType(0);
14169 bool IsAnd =
N->getOpcode() ==
ISD::AND;
14193 EVT VT =
N->getValueType(0);
14217 EVT VT =
N->getValueType(0);
14244 if (CondLHS != True)
14251 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
14263 if (!FalseRHSC || !FalseRHSC->
isZero())
14283 EVT VT =
N->getValueType(0);
14290 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
14317 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
14369 EVT VT =
N->getValueType(0);
14415 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
14440 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
14445 const APInt &Imm = ConstN00->getAPIntValue();
14446 if ((Imm + 1).isSignedIntN(12))
14467 EVT VT =
N->getValueType(0);
14479 const bool HasShlAdd =
14480 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
14494 for (
uint64_t Divisor : {3, 5, 9}) {
14495 if (MulAmt % Divisor != 0)
14497 uint64_t MulAmt2 = MulAmt / Divisor;
14504 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
14505 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
14522 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
14538 if (ScaleShift >= 1 && ScaleShift < 4) {
14539 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
14553 for (
uint64_t Divisor : {3, 5, 9}) {
14558 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
14569 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
14571 if (ScaleShift >= 1 && ScaleShift < 4) {
14572 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
14598 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
14600 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
14611 for (
uint64_t Divisor : {3, 5, 9}) {
14612 if (MulAmt % Divisor != 0)
14614 uint64_t MulAmt2 = MulAmt / Divisor;
14617 for (
uint64_t Divisor2 : {3, 5, 9}) {
14618 if (MulAmt2 % Divisor2 != 0)
14620 uint64_t MulAmt3 = MulAmt2 / Divisor2;
14643 EVT VT =
N->getValueType(0);
14650 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
14651 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
14664 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
14665 V3 != (HalfSize - 1))
14681 EVT VT =
N->getValueType(0);
14689 unsigned AddSubOpc;
14695 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
14696 AddSubOpc = V->getOpcode();
14698 SDValue Opnd = V->getOperand(1);
14699 MulOper = V->getOperand(0);
14708 if (IsAddSubWith1(N0)) {
14710 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
14713 if (IsAddSubWith1(N1)) {
14715 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
14730 if (isIndexTypeSigned(IndexType))
14733 if (!
N->hasOneUse())
14736 EVT VT =
N.getValueType();
14775 EVT SrcVT = Src.getValueType();
14779 NewElen = std::max(NewElen, 8U);
14802 EVT VT =
N->getValueType(0);
14805 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
14809 auto *N1C = dyn_cast<ConstantSDNode>(N1);
14821 if (!isIntEqualitySetCC(
Cond))
14830 const APInt &C1 = N1C->getAPIntValue();
14848 EVT VT =
N->getValueType(0);
14849 EVT SrcVT = cast<VTSDNode>(
N->getOperand(1))->
getVT();
14850 unsigned Opc = Src.getOpcode();
14855 Subtarget.hasStdExtZfhmin())
14857 Src.getOperand(0));
14861 VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
14864 Src.getOperand(1));
14872struct CombineResult;
14874enum ExtKind :
uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
14901struct NodeExtensionHelper {
14910 bool SupportsFPExt;
14913 bool EnforceOneUse;
14928 return OrigOperand;
14939 unsigned getExtOpc(ExtKind SupportsExt)
const {
14940 switch (SupportsExt) {
14941 case ExtKind::SExt:
14943 case ExtKind::ZExt:
14945 case ExtKind::FPExt:
14956 std::optional<ExtKind> SupportsExt)
const {
14957 if (!SupportsExt.has_value())
14958 return OrigOperand;
14960 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
14964 if (
Source.getValueType() == NarrowVT)
14968 if (
Source.getValueType().getVectorElementType() == MVT::bf16) {
14974 unsigned ExtOpc = getExtOpc(*SupportsExt);
14978 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
14985 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
14997 DAG.
getUNDEF(NarrowVT), Source, VL);
15010 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
15016 MVT EltVT = SupportsExt == ExtKind::FPExt
15018 :
MVT::getIntegerVT(NarrowSize);
15020 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
15021 "Trying to extend something we can't represent");
15028 static unsigned getSExtOpcode(
unsigned Opcode) {
15051 static unsigned getZExtOpcode(
unsigned Opcode) {
15077 static unsigned getFPExtOpcode(
unsigned Opcode) {
15102 static unsigned getSUOpcode(
unsigned Opcode) {
15104 "SU is only supported for MUL");
15110 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
15130 using CombineToTry = std::function<std::optional<CombineResult>(
15131 SDNode * ,
const NodeExtensionHelper & ,
15136 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
15140 unsigned Opc = OrigOperand.
getOpcode();
15144 "Unexpected Opcode");
15157 unsigned ScalarBits =
Op.getValueSizeInBits();
15159 if (ScalarBits < EltBits) {
15162 !Subtarget.
is64Bit() &&
"Unexpected splat");
15164 SupportsSExt =
true;
15168 SupportsZExt =
true;
15170 EnforceOneUse =
false;
15174 unsigned NarrowSize = EltBits / 2;
15177 if (NarrowSize < 8)
15181 SupportsSExt =
true;
15185 SupportsZExt =
true;
15187 EnforceOneUse =
false;
15190 bool isSupportedFPExtend(
SDNode *Root,
MVT NarrowEltVT,
15197 if (NarrowEltVT == MVT::bf16 && (!Subtarget.hasStdExtZvfbfwma() ||
15207 SupportsZExt =
false;
15208 SupportsSExt =
false;
15209 SupportsFPExt =
false;
15210 EnforceOneUse =
true;
15211 unsigned Opc = OrigOperand.
getOpcode();
15233 SupportsZExt =
true;
15236 SupportsSExt =
true;
15241 if (!isSupportedFPExtend(Root, NarrowEltVT, Subtarget))
15243 SupportsFPExt =
true;
15248 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
15260 if (!isSupportedFPExtend(Root,
Op.getOperand(0).getSimpleValueType(),
15265 unsigned ScalarBits =
Op.getOperand(0).getValueSizeInBits();
15266 if (NarrowSize != ScalarBits)
15269 SupportsFPExt =
true;
15278 static bool isSupportedRoot(
const SDNode *Root,
15307 Subtarget.hasStdExtZvbb();
15309 return Subtarget.hasStdExtZvbb();
15323 assert(isSupportedRoot(Root, Subtarget) &&
15324 "Trying to build an helper with an "
15325 "unsupported root");
15326 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
15342 if (OperandIdx == 1) {
15351 EnforceOneUse =
false;
15356 fillUpExtensionSupport(Root, DAG, Subtarget);
15362 static std::pair<SDValue, SDValue>
15365 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
15384 switch (
N->getOpcode()) {
15426struct CombineResult {
15428 unsigned TargetOpcode;
15430 std::optional<ExtKind> LHSExt;
15431 std::optional<ExtKind> RHSExt;
15435 NodeExtensionHelper
LHS;
15437 NodeExtensionHelper
RHS;
15439 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
15440 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
15441 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
15442 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
15451 std::tie(Mask, VL) =
15452 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
15466 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
15467 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
15468 Passthru, Mask, VL);
15482static std::optional<CombineResult>
15483canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
15484 const NodeExtensionHelper &RHS,
15487 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
15488 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
15489 Root, LHS, {ExtKind::ZExt}, RHS,
15491 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
15492 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
15493 Root, LHS, {ExtKind::SExt}, RHS,
15495 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
15496 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
15497 Root, LHS, {ExtKind::FPExt}, RHS,
15499 return std::nullopt;
15508static std::optional<CombineResult>
15509canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
15512 return canFoldToVWWithSameExtensionImpl(
15513 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
15521static std::optional<CombineResult>
15522canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
15525 if (
RHS.SupportsFPExt)
15526 return CombineResult(
15527 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
15528 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
15535 return CombineResult(
15536 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
15537 LHS, std::nullopt, RHS, {ExtKind::ZExt});
15539 return CombineResult(
15540 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
15541 LHS, std::nullopt, RHS, {ExtKind::SExt});
15542 return std::nullopt;
15549static std::optional<CombineResult>
15550canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15553 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
15561static std::optional<CombineResult>
15562canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15565 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
15573static std::optional<CombineResult>
15574canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
15577 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
15585static std::optional<CombineResult>
15586canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
15590 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
15591 return std::nullopt;
15592 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
15593 Root, LHS, {ExtKind::SExt}, RHS,
15598NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
15609 Strategies.
push_back(canFoldToVWWithSameExtension);
15618 Strategies.
push_back(canFoldToVWWithSameExtension);
15623 Strategies.
push_back(canFoldToVWWithSameExtension);
15630 Strategies.
push_back(canFoldToVWWithZEXT);
15635 Strategies.
push_back(canFoldToVWWithSEXT);
15640 Strategies.
push_back(canFoldToVWWithZEXT);
15645 Strategies.
push_back(canFoldToVWWithFPEXT);
15674 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
15680 Inserted.insert(
N);
15683 while (!Worklist.
empty()) {
15686 NodeExtensionHelper
LHS(Root, 0, DAG, Subtarget);
15687 NodeExtensionHelper
RHS(Root, 1, DAG, Subtarget);
15688 auto AppendUsersIfNeeded = [&Worklist, &Subtarget,
15689 &Inserted](
const NodeExtensionHelper &
Op) {
15690 if (
Op.needToPromoteOtherUsers()) {
15693 if (!NodeExtensionHelper::isSupportedRoot(TheUser, Subtarget))
15698 if (Inserted.insert(TheUser).second)
15711 NodeExtensionHelper::getSupportedFoldings(Root);
15713 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
15714 bool Matched =
false;
15715 for (
int Attempt = 0;
15716 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
15719 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
15720 FoldingStrategies) {
15721 std::optional<CombineResult> Res =
15722 FoldingStrategy(Root,
LHS,
RHS, DAG, Subtarget);
15729 if (Res->LHSExt.has_value())
15730 if (!AppendUsersIfNeeded(
LHS))
15732 if (Res->RHSExt.has_value())
15733 if (!AppendUsersIfNeeded(
RHS))
15745 SDValue InputRootReplacement;
15752 for (CombineResult Res : CombinesToApply) {
15753 SDValue NewValue = Res.materialize(DAG, Subtarget);
15754 if (!InputRootReplacement) {
15756 "First element is expected to be the current node");
15757 InputRootReplacement = NewValue;
15762 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
15766 return InputRootReplacement;
15773 unsigned Opc =
N->getOpcode();
15778 SDValue MergeOp =
N->getOperand(1);
15779 unsigned MergeOpc = MergeOp.
getOpcode();
15790 SDValue Passthru =
N->getOperand(2);
15804 Z = Z.getOperand(1);
15810 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
15817 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
15846 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
15852 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
15854 if (MemVT == MVT::i32)
15860 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
15895 if (!Subtarget.hasVendorXTHeadMemPair())
15907 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
15909 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
15910 return {
Ptr->getOperand(0), C1->getZExtValue()};
15914 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
15935 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
15938 if (Base1 != Base2)
15942 bool Valid =
false;
15943 if (MemVT == MVT::i32) {
15945 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
15947 }
else if (MemVT == MVT::i64) {
15949 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
15983 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15991 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
16001 EVT VT =
N->getValueType(0);
16004 MVT SrcVT = Src.getSimpleValueType();
16005 MVT SrcContainerVT = SrcVT;
16007 SDValue XVal = Src.getOperand(0);
16034 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
16038 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
16051 if (VT != MVT::i32 && VT != XLenVT)
16081 EVT DstVT =
N->getValueType(0);
16082 if (DstVT != XLenVT)
16088 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
16096 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
16099 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
16108 if (SatVT == DstVT)
16110 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
16116 Src = Src.getOperand(0);
16137 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
16143 EVT VT =
N->getValueType(0);
16196 unsigned Offset =
N->isTargetStrictFPOpcode();
16203 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
16205 V.getOperand(2) == VL) {
16207 V = V.getOperand(0);
16214 bool NegA = invertIfNegative(
A);
16215 bool NegB = invertIfNegative(
B);
16216 bool NegC = invertIfNegative(
C);
16219 if (!NegA && !NegB && !NegC)
16223 if (
N->isTargetStrictFPOpcode())
16225 {N->getOperand(0), A, B, C, Mask, VL});
16239 if (
N->isTargetStrictFPOpcode())
16249 EVT VT =
N->getValueType(0);
16254 if (!isa<ConstantSDNode>(
N->getOperand(1)))
16256 uint64_t ShAmt =
N->getConstantOperandVal(1);
16264 cast<VTSDNode>(N0.
getOperand(1))->getVT().getSizeInBits();
16269 if (LShAmt < ExtSize) {
16282 if (ShAmt > 32 || VT != MVT::i64)
16298 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
16311 !isa<ConstantSDNode>(U->getOperand(1)) ||
16312 U->getConstantOperandVal(1) > 32)
16367 if (!
Cond.hasOneUse())
16386 EVT VT =
Cond.getValueType();
16431 LHS =
LHS.getOperand(0);
16441 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
16445 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
16449 RHS =
LHS.getOperand(1);
16450 LHS =
LHS.getOperand(0);
16459 RHS =
LHS.getOperand(1);
16460 LHS =
LHS.getOperand(0);
16476 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
16517 bool Commutative =
true;
16518 unsigned Opc = TrueVal.getOpcode();
16526 Commutative =
false;
16534 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
16538 if (FalseVal == TrueVal.getOperand(0))
16540 else if (Commutative && FalseVal == TrueVal.getOperand(1))
16545 EVT VT =
N->getValueType(0);
16547 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
16553 assert(IdentityOperand &&
"No identity operand!");
16558 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
16559 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
16580 CountZeroes =
N->getOperand(2);
16581 ValOnZero =
N->getOperand(1);
16583 CountZeroes =
N->getOperand(1);
16584 ValOnZero =
N->getOperand(2);
16603 if (
Cond->getOperand(0) != CountZeroesArgument)
16619 CountZeroes, BitWidthMinusOne);
16629 EVT VT =
N->getValueType(0);
16630 EVT CondVT =
Cond.getValueType();
16638 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
16644 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
16665 SDValue TrueVal =
N->getOperand(1);
16666 SDValue FalseVal =
N->getOperand(2);
16681 EVT VT =
N->getValueType(0);
16688 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
16703 if (
Op.isUndef()) {
16716 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
16720 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
16721 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
16725 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
16753 const unsigned InVecOpcode = InVec->
getOpcode();
16763 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
16770 InVecLHS, InValLHS, EltNo);
16772 InVecRHS, InValRHS, EltNo);
16781 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
16784 unsigned Elt = IndexC->getZExtValue();
16792 unsigned ConcatOpIdx = Elt / ConcatNumElts;
16795 ConcatOp, InVal, NewIdx);
16799 ConcatOps[ConcatOpIdx] = ConcatOp;
16811 EVT VT =
N->getValueType(0);
16821 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
16823 !
SDValue(BaseLd, 0).hasOneUse())
16826 EVT BaseLdVT = BaseLd->getValueType(0);
16833 auto *Ld = dyn_cast<LoadSDNode>(
Op);
16834 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
16836 Ld->getValueType(0) != BaseLdVT)
16845 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
16847 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
16852 if (BIO1.equalBaseIndex(BIO2, DAG))
16857 SDValue P2 = Ld2->getBasePtr();
16863 return std::nullopt;
16867 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
16872 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
16873 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
16881 unsigned WideScalarBitWidth =
16894 auto [StrideVariant, MustNegateStride] = *BaseDiff;
16896 std::holds_alternative<SDValue>(StrideVariant)
16897 ? std::get<SDValue>(StrideVariant)
16900 if (MustNegateStride)
16908 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
16909 ConstStride && ConstStride->getSExtValue() >= 0)
16913 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
16919 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
16923 WideVecVT,
DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
16938 if (
N->getValueType(0).isFixedLengthVector())
16941 SDValue Addend =
N->getOperand(0);
16945 SDValue AddPassthruOp =
N->getOperand(2);
16946 if (!AddPassthruOp.
isUndef())
16950 auto IsVWMulOpc = [](
unsigned Opc) {
16969 if (!MulPassthruOp.
isUndef())
16979 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
16980 }(
N, DAG, Subtarget);
16985 if (AddMask != MulMask || AddVL != MulVL)
16990 "Unexpected opcode after VWMACC_VL");
16992 "Unexpected opcode after VWMACC_VL!");
16994 "Unexpected opcode after VWMUL_VL!");
16996 "Unexpected opcode after VWMUL_VL!");
16999 EVT VT =
N->getValueType(0);
17015 const EVT IndexVT = Index.getValueType();
17019 if (!isIndexTypeSigned(IndexType))
17051 for (
unsigned i = 0; i < Index->getNumOperands(); i++) {
17054 if (Index->getOperand(i)->isUndef())
17056 uint64_t C = Index->getConstantOperandVal(i);
17057 if (
C % ElementSize != 0)
17059 C =
C / ElementSize;
17063 ActiveLanes.
set(
C);
17065 return ActiveLanes.
all();
17083 if (NumElems % 2 != 0)
17087 const unsigned WiderElementSize = ElementSize * 2;
17088 if (WiderElementSize > ST.getELen()/8)
17091 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
17094 for (
unsigned i = 0; i < Index->getNumOperands(); i++) {
17097 if (Index->getOperand(i)->isUndef())
17101 uint64_t C = Index->getConstantOperandVal(i);
17103 if (
C % WiderElementSize != 0)
17108 if (
C !=
Last + ElementSize)
17125 (isa<RegisterSDNode>(VL) &&
17126 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
17128 Mask.getOperand(0) != VL)
17131 auto IsTruncNode = [&](
SDValue V) {
17133 V.getOperand(1) == Mask && V.getOperand(2) == VL;
17140 while (IsTruncNode(
Op)) {
17141 if (!
Op.hasOneUse())
17143 Op =
Op.getOperand(0);
17178 MVT VT =
N->getSimpleValueType(0);
17183 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
17185 if (V.getOpcode() != Opc &&
17186 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
17187 V.getOperand(3) == Mask && V.getOperand(4) == VL))
17195 Op.getOperand(1).getValueType().isFixedLengthVector() &&
17197 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
17199 Op =
Op.getOperand(1).getOperand(0);
17202 return V.getOperand(0);
17205 Op.getOperand(2) == VL) {
17206 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
17208 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
17209 return V.getOperand(0);
17218 auto DetectUSatPattern = [&](
SDValue V) {
17243 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
17249 auto DetectSSatPattern = [&](
SDValue V) {
17251 unsigned NumSrcBits = V.getScalarValueSizeInBits();
17259 if (HiC == SignedMax && LoC == SignedMin)
17265 if (HiC == SignedMax && LoC == SignedMin)
17275 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
17277 Src = Src.getOperand(0);
17281 if ((Val = DetectUSatPattern(Src)))
17283 else if ((Val = DetectSSatPattern(Src)))
17293 Val = DAG.
getNode(ClipOpc,
DL, ValVT, Val, Mask, VL);
17294 }
while (ValVT != VT);
17308 EVT VT =
N->getValueType(0);
17316 Src = Src.getOperand(0);
17321 Src = Src.getOperand(0);
17322 EVT SrcEVT = Src.getValueType();
17354 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
17365 switch (
N->getOpcode()) {
17385 APInt V =
C->getValueAPF().bitcastToAPInt();
17420 if (SimplifyDemandedLowBitsHelper(0, 32) ||
17421 SimplifyDemandedLowBitsHelper(1, 5))
17429 if (SimplifyDemandedLowBitsHelper(0, 32))
17446 MVT VT =
N->getSimpleValueType(0);
17449 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(Op0)) {
17462 "Unexpected value type!");
17467 cast<LoadSDNode>(Op0)->isSimple()) {
17469 auto *LN0 = cast<LoadSDNode>(Op0);
17472 LN0->getBasePtr(), IVT, LN0->getMemOperand());
17496 EVT VT =
N->getValueType(0);
17549 if (!
C || !
C->getValueAPF().isExactlyValue(+1.0))
17551 EVT VT =
N->getValueType(0);
17580 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
17585 Src.getOperand(0));
17590 Src.getOperand(0), Src.getOperand(1));
17611 unsigned Opc =
N->getOpcode();
17626 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
17636 N->getValueType(0), Val,
Cond.getOperand(0));
17647 SDValue FalseV =
N->getOperand(4);
17649 EVT VT =
N->getValueType(0);
17652 if (TrueV == FalseV)
17657 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
17663 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
17664 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
17667 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
17668 isInt<12>(TrueSImm - FalseSImm)) {
17684 {LHS, RHS, CC, TrueV, FalseV});
17751 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
17764 EVT VT =
N->getValueType(0);
17788 const auto *MGN = cast<MaskedGatherSDNode>(
N);
17789 const EVT VT =
N->getValueType(0);
17790 SDValue Index = MGN->getIndex();
17791 SDValue ScaleOp = MGN->getScale();
17793 assert(!MGN->isIndexScaled() &&
17794 "Scaled gather/scatter should not be formed");
17799 N->getVTList(), MGN->getMemoryVT(),
DL,
17800 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17801 MGN->getBasePtr(), Index, ScaleOp},
17802 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17806 N->getVTList(), MGN->getMemoryVT(),
DL,
17807 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17808 MGN->getBasePtr(), Index, ScaleOp},
17809 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17815 if (std::optional<VIDSequence> SimpleVID =
17817 SimpleVID && SimpleVID->StepDenominator == 1) {
17818 const int64_t StepNumerator = SimpleVID->StepNumerator;
17819 const int64_t Addend = SimpleVID->Addend;
17826 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
17834 VT,
DL, MGN->getChain(), BasePtr,
17836 EVL, MGN->getMemOperand());
17838 StridedLoad, MGN->getPassThru(), EVL);
17848 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
17850 MGN->getMemoryVT(), MGN->getMemOperand(),
17859 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
17861 for (
unsigned i = 0; i < Index->getNumOperands(); i += 2)
17862 NewIndices.
push_back(Index.getOperand(i));
17863 EVT IndexVT = Index.getValueType()
17864 .getHalfNumVectorElementsVT(*DAG.
getContext());
17870 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
17872 EltCnt.divideCoefficientBy(2));
17875 EltCnt.divideCoefficientBy(2));
17880 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
17889 const auto *MSN = cast<MaskedScatterSDNode>(
N);
17890 SDValue Index = MSN->getIndex();
17891 SDValue ScaleOp = MSN->getScale();
17893 assert(!MSN->isIndexScaled() &&
17894 "Scaled gather/scatter should not be formed");
17899 N->getVTList(), MSN->getMemoryVT(),
DL,
17900 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17902 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17906 N->getVTList(), MSN->getMemoryVT(),
DL,
17907 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17909 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17911 EVT VT = MSN->getValue()->getValueType(0);
17913 if (!MSN->isTruncatingStore() &&
17917 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
17918 DAG.
getUNDEF(XLenVT), MSN->getMask(),
17919 MSN->getMemoryVT(), MSN->getMemOperand(),
17924 case ISD::VP_GATHER: {
17925 const auto *VPGN = cast<VPGatherSDNode>(
N);
17926 SDValue Index = VPGN->getIndex();
17927 SDValue ScaleOp = VPGN->getScale();
17929 assert(!VPGN->isIndexScaled() &&
17930 "Scaled gather/scatter should not be formed");
17935 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17936 ScaleOp, VPGN->getMask(),
17937 VPGN->getVectorLength()},
17938 VPGN->getMemOperand(), IndexType);
17942 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17943 ScaleOp, VPGN->getMask(),
17944 VPGN->getVectorLength()},
17945 VPGN->getMemOperand(), IndexType);
17949 case ISD::VP_SCATTER: {
17950 const auto *VPSN = cast<VPScatterSDNode>(
N);
17951 SDValue Index = VPSN->getIndex();
17952 SDValue ScaleOp = VPSN->getScale();
17954 assert(!VPSN->isIndexScaled() &&
17955 "Scaled gather/scatter should not be formed");
17960 {VPSN->getChain(), VPSN->getValue(),
17961 VPSN->getBasePtr(), Index, ScaleOp,
17962 VPSN->getMask(), VPSN->getVectorLength()},
17963 VPSN->getMemOperand(), IndexType);
17967 {VPSN->getChain(), VPSN->getValue(),
17968 VPSN->getBasePtr(), Index, ScaleOp,
17969 VPSN->getMask(), VPSN->getVectorLength()},
17970 VPSN->getMemOperand(), IndexType);
17984 EVT VT =
N->getValueType(0);
17987 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
17988 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
18006 EVT VT =
N->getValueType(0);
18010 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
18050 auto *Store = cast<StoreSDNode>(
N);
18051 SDValue Chain = Store->getChain();
18052 EVT MemVT = Store->getMemoryVT();
18053 SDValue Val = Store->getValue();
18056 bool IsScalarizable =
18058 Store->isSimple() &&
18088 NewVT, *Store->getMemOperand())) {
18090 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
18091 Store->getPointerInfo(), Store->getOriginalAlign(),
18092 Store->getMemOperand()->getFlags());
18100 if (
auto *L = dyn_cast<LoadSDNode>(Val);
18102 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
18104 L->getMemoryVT() == MemVT) {
18107 NewVT, *Store->getMemOperand()) &&
18109 NewVT, *L->getMemOperand())) {
18111 L->getPointerInfo(), L->getOriginalAlign(),
18112 L->getMemOperand()->getFlags());
18113 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
18114 Store->getPointerInfo(), Store->getOriginalAlign(),
18115 Store->getMemOperand()->getFlags());
18127 MVT VecVT = Src.getSimpleValueType();
18134 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
18137 Store->getMemOperand(), Store->getAddressingMode(),
18138 Store->isTruncatingStore(),
false);
18145 EVT VT =
N->getValueType(0);
18167 const MVT VT =
N->getSimpleValueType(0);
18168 SDValue Passthru =
N->getOperand(0);
18169 SDValue Scalar =
N->getOperand(1);
18178 const MVT VT =
N->getSimpleValueType(0);
18179 SDValue Passthru =
N->getOperand(0);
18180 SDValue Scalar =
N->getOperand(1);
18185 unsigned ScalarSize = Scalar.getValueSizeInBits();
18187 if (ScalarSize > EltWidth && Passthru.
isUndef())
18188 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
18195 (!Const || Const->isZero() ||
18196 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
18206 if (
N->getOperand(0).isUndef() &&
18209 Src.getOperand(0).getValueType().isScalableVector()) {
18210 EVT VT =
N->getValueType(0);
18211 EVT SrcVT = Src.getOperand(0).getValueType();
18215 return Src.getOperand(0);
18221 const MVT VT =
N->getSimpleValueType(0);
18222 SDValue Passthru =
N->getOperand(0);
18223 SDValue Scalar =
N->getOperand(1);
18227 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
18228 return Scalar.getOperand(0);
18237 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
18247 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
18255 MVT VecVT =
N->getOperand(0).getSimpleValueType();
18257 if (M1VT.
bitsLT(VecVT)) {
18268 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
18273 case Intrinsic::riscv_vcpop:
18274 case Intrinsic::riscv_vcpop_mask:
18275 case Intrinsic::riscv_vfirst:
18276 case Intrinsic::riscv_vfirst_mask: {
18278 if (IntNo == Intrinsic::riscv_vcpop_mask ||
18279 IntNo == Intrinsic::riscv_vfirst_mask)
18280 VL =
N->getOperand(3);
18285 EVT VT =
N->getValueType(0);
18286 if (IntNo == Intrinsic::riscv_vfirst ||
18287 IntNo == Intrinsic::riscv_vfirst_mask)
18296 EVT VT =
N->getValueType(0);
18307 for (
unsigned i = 0; i < NF; ++i)
18314 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
18337 EVT XVT,
unsigned KeptBits)
const {
18342 if (XVT != MVT::i32 && XVT != MVT::i64)
18346 if (KeptBits == 32 || KeptBits == 64)
18350 return Subtarget.hasStdExtZbb() &&
18351 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
18359 "Expected shift op");
18377 if (!isa<StoreSDNode>(
Use) && !isa<LoadSDNode>(
Use))
18386 return isUsedByLdSt(N0.
getNode(),
N);
18388 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
18389 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
18391 const APInt &C1Int = C1->getAPIntValue();
18392 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
18418 if (C1Cost < ShiftedC1Cost)
18441 EVT VT =
Op.getValueType();
18445 unsigned Opcode =
Op.getOpcode();
18453 const APInt &Mask =
C->getAPIntValue();
18462 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
18463 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
18465 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
18466 if (NewMask == Mask)
18471 Op.getOperand(0), NewC);
18484 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
18485 if (IsLegalMask(NewMask))
18486 return UseMask(NewMask);
18489 if (VT == MVT::i64) {
18491 if (IsLegalMask(NewMask))
18492 return UseMask(NewMask);
18507 APInt NewMask = ShrunkMask;
18508 if (MinSignedBits <= 12)
18510 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
18516 assert(IsLegalMask(NewMask));
18517 return UseMask(NewMask);
18521 static const uint64_t GREVMasks[] = {
18522 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
18523 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
18525 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
18526 unsigned Shift = 1 << Stage;
18527 if (ShAmt & Shift) {
18529 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
18541 const APInt &DemandedElts,
18543 unsigned Depth)
const {
18545 unsigned Opc =
Op.getOpcode();
18550 "Should use MaskedValueIsZero if you don't know whether Op"
18551 " is a target node!");
18634 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
18637 if (MaxVLenB == MinVLenB)
18654 case Intrinsic::riscv_vsetvli:
18655 case Intrinsic::riscv_vsetvlimax: {
18656 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
18657 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
18663 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
18666 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
18667 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
18669 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
18682 unsigned Depth)
const {
18683 switch (
Op.getOpcode()) {
18689 if (Tmp == 1)
return 1;
18692 return std::min(Tmp, Tmp2);
18704 if (Tmp < 33)
return 1;
18729 unsigned XLen = Subtarget.
getXLen();
18730 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
18731 if (EltBits <= XLen)
18732 return XLen - EltBits + 1;
18736 unsigned IntNo =
Op.getConstantOperandVal(1);
18740 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
18741 case Intrinsic::riscv_masked_atomicrmw_add_i64:
18742 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
18743 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
18744 case Intrinsic::riscv_masked_atomicrmw_max_i64:
18745 case Intrinsic::riscv_masked_atomicrmw_min_i64:
18746 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
18747 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
18748 case Intrinsic::riscv_masked_cmpxchg_i64:
18756 assert(Subtarget.hasStdExtA());
18771 switch (
Op.getOpcode()) {
18777 return !
Op.getValueType().isInteger();
18785 assert(Ld &&
"Unexpected null LoadSDNode");
18793 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
18794 if (!CNode || CNode->isMachineConstantPoolEntry() ||
18795 CNode->getOffset() != 0)
18803 auto *CNode = GetSupportedConstantPool(
Ptr);
18804 if (!CNode || CNode->getTargetFlags() != 0)
18807 return CNode->getConstVal();
18815 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
18816 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
18822 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
18825 return CNodeLo->getConstVal();
18830 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
18862 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18865 int64_t LoCounter =
MI.getOperand(2).getImm();
18866 int64_t HiCounter =
MI.getOperand(3).getImm();
18876 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
18888 MI.eraseFromParent();
18896 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
18904 Register SrcReg =
MI.getOperand(2).getReg();
18924 MI.eraseFromParent();
18931 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
18932 "Unexpected instruction");
18938 Register DstReg =
MI.getOperand(0).getReg();
18961 MI.eraseFromParent();
18966 switch (
MI.getOpcode()) {
18969 case RISCV::Select_GPR_Using_CC_GPR:
18970 case RISCV::Select_GPR_Using_CC_Imm:
18971 case RISCV::Select_FPR16_Using_CC_GPR:
18972 case RISCV::Select_FPR16INX_Using_CC_GPR:
18973 case RISCV::Select_FPR32_Using_CC_GPR:
18974 case RISCV::Select_FPR32INX_Using_CC_GPR:
18975 case RISCV::Select_FPR64_Using_CC_GPR:
18976 case RISCV::Select_FPR64INX_Using_CC_GPR:
18977 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18983 unsigned RelOpcode,
unsigned EqOpcode,
18986 Register DstReg =
MI.getOperand(0).getReg();
18987 Register Src1Reg =
MI.getOperand(1).getReg();
18988 Register Src2Reg =
MI.getOperand(2).getReg();
18990 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19014 MI.eraseFromParent();
19065 F->insert(It, FirstMBB);
19066 F->insert(It, SecondMBB);
19067 F->insert(It, SinkMBB);
19116 First.eraseFromParent();
19155 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
19156 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
19157 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
19158 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
19159 Next->getOperand(5).isKill())
19164 if (
MI.getOperand(2).isReg())
19165 RHS =
MI.getOperand(2).getReg();
19170 SelectDests.
insert(
MI.getOperand(0).getReg());
19174 SequenceMBBI != E; ++SequenceMBBI) {
19175 if (SequenceMBBI->isDebugInstr())
19178 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
19179 !SequenceMBBI->getOperand(2).isReg() ||
19180 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
19181 SequenceMBBI->getOperand(3).getImm() !=
CC ||
19182 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
19183 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
19185 LastSelectPseudo = &*SequenceMBBI;
19187 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
19190 if (SequenceMBBI->hasUnmodeledSideEffects() ||
19191 SequenceMBBI->mayLoadOrStore() ||
19192 SequenceMBBI->usesCustomInsertionHook())
19195 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
19210 F->insert(
I, IfFalseMBB);
19211 F->insert(
I, TailMBB);
19214 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
19220 TailMBB->
push_back(DebugInstr->removeFromParent());
19224 TailMBB->
splice(TailMBB->
end(), HeadMBB,
19234 if (
MI.getOperand(2).isImm())
19237 .
addImm(
MI.getOperand(2).getImm())
19249 auto SelectMBBI =
MI.getIterator();
19250 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
19252 while (SelectMBBI != SelectEnd) {
19253 auto Next = std::next(SelectMBBI);
19257 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
19258 .
addReg(SelectMBBI->getOperand(4).getReg())
19260 .
addReg(SelectMBBI->getOperand(5).getReg())
19275 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
19276 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
19278 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
19279 assert(
Masked &&
"Could not find masked instruction for LMUL and SEW pair");
19285 unsigned CVTXOpc) {
19291 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19304 .
add(
MI.getOperand(1))
19305 .
add(
MI.getOperand(2))
19306 .
add(
MI.getOperand(3))
19308 .
add(
MI.getOperand(4))
19309 .
add(
MI.getOperand(5))
19310 .
add(
MI.getOperand(6))
19325 .
add(
MI.getOperand(0))
19326 .
add(
MI.getOperand(1))
19328 .
add(
MI.getOperand(3))
19330 .
add(
MI.getOperand(4))
19331 .
add(
MI.getOperand(5))
19332 .
add(
MI.getOperand(6))
19342 MI.eraseFromParent();
19348 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
19350 switch (
MI.getOpcode()) {
19353 case RISCV::PseudoFROUND_H:
19354 CmpOpc = RISCV::FLT_H;
19355 F2IOpc = RISCV::FCVT_W_H;
19356 I2FOpc = RISCV::FCVT_H_W;
19357 FSGNJOpc = RISCV::FSGNJ_H;
19358 FSGNJXOpc = RISCV::FSGNJX_H;
19359 RC = &RISCV::FPR16RegClass;
19361 case RISCV::PseudoFROUND_H_INX:
19362 CmpOpc = RISCV::FLT_H_INX;
19363 F2IOpc = RISCV::FCVT_W_H_INX;
19364 I2FOpc = RISCV::FCVT_H_W_INX;
19365 FSGNJOpc = RISCV::FSGNJ_H_INX;
19366 FSGNJXOpc = RISCV::FSGNJX_H_INX;
19367 RC = &RISCV::GPRF16RegClass;
19369 case RISCV::PseudoFROUND_S:
19370 CmpOpc = RISCV::FLT_S;
19371 F2IOpc = RISCV::FCVT_W_S;
19372 I2FOpc = RISCV::FCVT_S_W;
19373 FSGNJOpc = RISCV::FSGNJ_S;
19374 FSGNJXOpc = RISCV::FSGNJX_S;
19375 RC = &RISCV::FPR32RegClass;
19377 case RISCV::PseudoFROUND_S_INX:
19378 CmpOpc = RISCV::FLT_S_INX;
19379 F2IOpc = RISCV::FCVT_W_S_INX;
19380 I2FOpc = RISCV::FCVT_S_W_INX;
19381 FSGNJOpc = RISCV::FSGNJ_S_INX;
19382 FSGNJXOpc = RISCV::FSGNJX_S_INX;
19383 RC = &RISCV::GPRF32RegClass;
19385 case RISCV::PseudoFROUND_D:
19387 CmpOpc = RISCV::FLT_D;
19388 F2IOpc = RISCV::FCVT_L_D;
19389 I2FOpc = RISCV::FCVT_D_L;
19390 FSGNJOpc = RISCV::FSGNJ_D;
19391 FSGNJXOpc = RISCV::FSGNJX_D;
19392 RC = &RISCV::FPR64RegClass;
19394 case RISCV::PseudoFROUND_D_INX:
19396 CmpOpc = RISCV::FLT_D_INX;
19397 F2IOpc = RISCV::FCVT_L_D_INX;
19398 I2FOpc = RISCV::FCVT_D_L_INX;
19399 FSGNJOpc = RISCV::FSGNJ_D_INX;
19400 FSGNJXOpc = RISCV::FSGNJX_D_INX;
19401 RC = &RISCV::GPRRegClass;
19413 F->insert(
I, CvtMBB);
19414 F->insert(
I, DoneMBB);
19425 Register DstReg =
MI.getOperand(0).getReg();
19426 Register SrcReg =
MI.getOperand(1).getReg();
19427 Register MaxReg =
MI.getOperand(2).getReg();
19428 int64_t FRM =
MI.getOperand(3).getImm();
19433 Register FabsReg =
MRI.createVirtualRegister(RC);
19437 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19452 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
19474 MI.eraseFromParent();
19481 switch (
MI.getOpcode()) {
19484 case RISCV::ReadCounterWide:
19486 "ReadCounterWide is only to be used on riscv32");
19488 case RISCV::Select_GPR_Using_CC_GPR:
19489 case RISCV::Select_GPR_Using_CC_Imm:
19490 case RISCV::Select_FPR16_Using_CC_GPR:
19491 case RISCV::Select_FPR16INX_Using_CC_GPR:
19492 case RISCV::Select_FPR32_Using_CC_GPR:
19493 case RISCV::Select_FPR32INX_Using_CC_GPR:
19494 case RISCV::Select_FPR64_Using_CC_GPR:
19495 case RISCV::Select_FPR64INX_Using_CC_GPR:
19496 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
19498 case RISCV::BuildPairF64Pseudo:
19500 case RISCV::SplitF64Pseudo:
19502 case RISCV::PseudoQuietFLE_H:
19504 case RISCV::PseudoQuietFLE_H_INX:
19505 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
19506 case RISCV::PseudoQuietFLT_H:
19508 case RISCV::PseudoQuietFLT_H_INX:
19509 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
19510 case RISCV::PseudoQuietFLE_S:
19512 case RISCV::PseudoQuietFLE_S_INX:
19513 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
19514 case RISCV::PseudoQuietFLT_S:
19516 case RISCV::PseudoQuietFLT_S_INX:
19517 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
19518 case RISCV::PseudoQuietFLE_D:
19520 case RISCV::PseudoQuietFLE_D_INX:
19521 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
19522 case RISCV::PseudoQuietFLE_D_IN32X:
19525 case RISCV::PseudoQuietFLT_D:
19527 case RISCV::PseudoQuietFLT_D_INX:
19528 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
19529 case RISCV::PseudoQuietFLT_D_IN32X:
19533 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
19535 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
19537 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
19539 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
19541 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
19543 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
19545 case RISCV::PseudoFROUND_H:
19546 case RISCV::PseudoFROUND_H_INX:
19547 case RISCV::PseudoFROUND_S:
19548 case RISCV::PseudoFROUND_S_INX:
19549 case RISCV::PseudoFROUND_D:
19550 case RISCV::PseudoFROUND_D_INX:
19551 case RISCV::PseudoFROUND_D_IN32X:
19553 case TargetOpcode::STATEPOINT:
19559 MI.addOperand(*
MI.getMF(),
19565 case TargetOpcode::STACKMAP:
19566 case TargetOpcode::PATCHPOINT:
19569 "supported on 64-bit targets");
19587 if (
MI.readsRegister(RISCV::FRM,
nullptr))
19593void RISCVTargetLowering::analyzeInputArgs(
19597 unsigned NumArgs = Ins.size();
19600 for (
unsigned i = 0; i != NumArgs; ++i) {
19601 MVT ArgVT = Ins[i].VT;
19604 Type *ArgTy =
nullptr;
19607 else if (Ins[i].isOrigArg())
19608 ArgTy = FType->
getParamType(Ins[i].getOrigArgIndex());
19611 true, IsRet, ArgTy)) {
19612 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
19619void RISCVTargetLowering::analyzeOutputArgs(
19623 unsigned NumArgs = Outs.
size();
19625 for (
unsigned i = 0; i != NumArgs; i++) {
19626 MVT ArgVT = Outs[i].VT;
19628 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
19631 Outs[i].IsFixed, IsRet, OrigTy)) {
19632 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
19683 if (In.isOrigArg()) {
19688 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
19689 (
BitWidth < 32 && In.Flags.isZExt())) {
19711 if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32)
19760 ExtType,
DL, LocVT, Chain, FIN,
19777 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19790 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19805 switch (CallConv) {
19815 if (Subtarget.hasStdExtE())
19819 "(Zdinx/D) instruction set extensions");
19823 if (Func.hasFnAttribute(
"interrupt")) {
19824 if (!Func.arg_empty())
19826 "Functions with the interrupt attribute cannot have arguments!");
19831 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19833 "Function interrupt attribute argument not supported!");
19838 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19840 std::vector<SDValue> OutChains;
19849 analyzeInputArgs(MF, CCInfo, Ins,
false,
19853 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19874 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19875 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19877 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19879 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19908 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19913 if (VarArgsSaveSize == 0) {
19917 int VaArgOffset = -VarArgsSaveSize;
19925 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19926 VarArgsSaveSize += XLenInBytes;
19933 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19938 Chain,
DL, ArgValue, FIN,
19940 OutChains.push_back(Store);
19954 if (!OutChains.empty()) {
19955 OutChains.push_back(Chain);
19965bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19969 auto CalleeCC = CLI.CallConv;
19970 auto &Outs = CLI.Outs;
19972 auto CallerCC = Caller.getCallingConv();
19979 if (Caller.hasFnAttribute(
"interrupt"))
19994 for (
auto &VA : ArgLocs)
20000 auto IsCallerStructRet = Caller.hasStructRetAttr();
20001 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
20002 if (IsCallerStructRet || IsCalleeStructRet)
20007 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
20008 if (CalleeCC != CallerCC) {
20009 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
20010 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
20017 for (
auto &Arg : Outs)
20018 if (Arg.Flags.isByVal())
20053 if (Subtarget.hasStdExtE())
20057 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
20063 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
20069 "site marked musttail");
20076 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20078 if (!Flags.isByVal())
20082 unsigned Size = Flags.getByValSize();
20083 Align Alignment = Flags.getNonZeroByValAlign();
20090 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
20092 false,
nullptr, IsTailCall,
20104 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
20107 SDValue ArgValue = OutVals[OutIdx];
20127 if (!StackPtr.getNode())
20139 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
20157 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
20158 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
20164 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
20165 SDValue PartValue = OutVals[OutIdx + 1];
20166 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
20178 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
20180 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
20182 for (
const auto &Part : Parts) {
20183 SDValue PartValue = Part.first;
20184 SDValue PartOffset = Part.second;
20191 ArgValue = SpillSlot;
20197 if (Flags.isByVal())
20198 ArgValue = ByValArgs[j++];
20205 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
20206 "for passing parameters");
20209 if (!StackPtr.getNode())
20223 if (!MemOpChains.
empty())
20229 for (
auto &Reg : RegsToPass) {
20230 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
20237 validateCCReservedRegs(RegsToPass, MF);
20241 "Return address register required, but has been reserved."});
20246 bool CalleeIsLargeExternalSymbol =
false;
20248 if (
auto *S = dyn_cast<GlobalAddressSDNode>(Callee))
20250 else if (
auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
20252 CalleeIsLargeExternalSymbol =
true;
20268 for (
auto &Reg : RegsToPass)
20274 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
20275 assert(Mask &&
"Missing call preserved mask for calling convention");
20284 "Unexpected CFI type for a direct call");
20292 bool NeedSWGuarded =
false;
20294 Subtarget.hasStdExtZicfilp() &&
20296 NeedSWGuarded =
true;
20310 Chain = DAG.
getNode(CallOpc,
DL, NodeTys, Ops);
20323 analyzeInputArgs(MF, RetCCInfo, Ins,
true,
CC_RISCV);
20326 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
20327 auto &VA = RVLocs[i];
20335 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
20336 assert(VA.needsCustom());
20356 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
20358 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20359 MVT VT = Outs[i].VT;
20362 true,
true,
nullptr))
20394 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
20395 SDValue Val = OutVals[OutIdx];
20404 DAG.
getVTList(MVT::i32, MVT::i32), Val);
20408 Register RegHi = RVLocs[++i].getLocReg();
20414 "Return value register required, but has been reserved."});
20430 "Return value register required, but has been reserved."});
20452 if (Func.hasFnAttribute(
"interrupt")) {
20453 if (!Func.getReturnType()->isVoidTy())
20455 "Functions with the interrupt attribute must have void return type!");
20461 if (Kind ==
"supervisor")
20467 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
20470void RISCVTargetLowering::validateCCReservedRegs(
20471 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
20480 F,
"Argument register required, but has been reserved."});
20486 if (
N->getNumValues() != 1)
20488 if (!
N->hasNUsesOfValue(1, 0))
20491 SDNode *Copy = *
N->user_begin();
20505 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20509 bool HasRet =
false;
20510 for (
SDNode *Node : Copy->users()) {
20518 Chain = Copy->getOperand(0);
20527#define NODE_NAME_CASE(NODE) \
20528 case RISCVISD::NODE: \
20529 return "RISCVISD::" #NODE;
20790#undef NODE_NAME_CASE
20797 if (Constraint.
size() == 1) {
20798 switch (Constraint[0]) {
20815 if (Constraint ==
"vr" || Constraint ==
"vd" || Constraint ==
"vm")
20817 if (Constraint ==
"cr" || Constraint ==
"cf")
20823std::pair<unsigned, const TargetRegisterClass *>
20829 if (Constraint.
size() == 1) {
20830 switch (Constraint[0]) {
20835 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20836 return std::make_pair(0U, &RISCV::GPRF16NoX0RegClass);
20837 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20838 return std::make_pair(0U, &RISCV::GPRF32NoX0RegClass);
20839 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20840 return std::make_pair(0U, &RISCV::GPRPairNoX0RegClass);
20841 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20843 if (VT == MVT::f16) {
20844 if (Subtarget.hasStdExtZfhmin())
20845 return std::make_pair(0U, &RISCV::FPR16RegClass);
20846 if (Subtarget.hasStdExtZhinxmin())
20847 return std::make_pair(0U, &RISCV::GPRF16NoX0RegClass);
20848 }
else if (VT == MVT::f32) {
20849 if (Subtarget.hasStdExtF())
20850 return std::make_pair(0U, &RISCV::FPR32RegClass);
20851 if (Subtarget.hasStdExtZfinx())
20852 return std::make_pair(0U, &RISCV::GPRF32NoX0RegClass);
20853 }
else if (VT == MVT::f64) {
20854 if (Subtarget.hasStdExtD())
20855 return std::make_pair(0U, &RISCV::FPR64RegClass);
20856 if (Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20857 return std::make_pair(0U, &RISCV::GPRPairNoX0RegClass);
20858 if (Subtarget.hasStdExtZdinx() && Subtarget.
is64Bit())
20859 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20863 if (VT == MVT::f64 && !Subtarget.
is64Bit() && Subtarget.hasStdExtZdinx())
20864 return std::make_pair(0U, &RISCV::GPRPairNoX0RegClass);
20865 return std::make_pair(0U, &RISCV::GPRPairNoX0RegClass);
20869 }
else if (Constraint ==
"vr") {
20870 for (
const auto *RC :
20871 {&RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
20872 &RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN3M1RegClass,
20873 &RISCV::VRN4M1RegClass, &RISCV::VRN5M1RegClass,
20874 &RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass,
20875 &RISCV::VRN8M1RegClass, &RISCV::VRN2M2RegClass,
20876 &RISCV::VRN3M2RegClass, &RISCV::VRN4M2RegClass,
20877 &RISCV::VRN2M4RegClass}) {
20879 return std::make_pair(0U, RC);
20881 }
else if (Constraint ==
"vd") {
20882 for (
const auto *RC :
20883 {&RISCV::VRNoV0RegClass, &RISCV::VRM2NoV0RegClass,
20884 &RISCV::VRM4NoV0RegClass, &RISCV::VRM8NoV0RegClass,
20885 &RISCV::VRN2M1NoV0RegClass, &RISCV::VRN3M1NoV0RegClass,
20886 &RISCV::VRN4M1NoV0RegClass, &RISCV::VRN5M1NoV0RegClass,
20887 &RISCV::VRN6M1NoV0RegClass, &RISCV::VRN7M1NoV0RegClass,
20888 &RISCV::VRN8M1NoV0RegClass, &RISCV::VRN2M2NoV0RegClass,
20889 &RISCV::VRN3M2NoV0RegClass, &RISCV::VRN4M2NoV0RegClass,
20890 &RISCV::VRN2M4NoV0RegClass}) {
20892 return std::make_pair(0U, RC);
20894 }
else if (Constraint ==
"vm") {
20895 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20896 return std::make_pair(0U, &RISCV::VMV0RegClass);
20897 }
else if (Constraint ==
"cr") {
20898 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20899 return std::make_pair(0U, &RISCV::GPRF16CRegClass);
20900 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20901 return std::make_pair(0U, &RISCV::GPRF32CRegClass);
20902 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20903 return std::make_pair(0U, &RISCV::GPRPairCRegClass);
20905 return std::make_pair(0U, &RISCV::GPRCRegClass);
20906 }
else if (Constraint ==
"cf") {
20907 if (VT == MVT::f16) {
20908 if (Subtarget.hasStdExtZfhmin())
20909 return std::make_pair(0U, &RISCV::FPR16CRegClass);
20910 if (Subtarget.hasStdExtZhinxmin())
20911 return std::make_pair(0U, &RISCV::GPRF16CRegClass);
20912 }
else if (VT == MVT::f32) {
20913 if (Subtarget.hasStdExtF())
20914 return std::make_pair(0U, &RISCV::FPR32CRegClass);
20915 if (Subtarget.hasStdExtZfinx())
20916 return std::make_pair(0U, &RISCV::GPRF32CRegClass);
20917 }
else if (VT == MVT::f64) {
20918 if (Subtarget.hasStdExtD())
20919 return std::make_pair(0U, &RISCV::FPR64CRegClass);
20920 if (Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20921 return std::make_pair(0U, &RISCV::GPRPairCRegClass);
20922 if (Subtarget.hasStdExtZdinx() && Subtarget.
is64Bit())
20923 return std::make_pair(0U, &RISCV::GPRCRegClass);
20932 .
Case(
"{zero}", RISCV::X0)
20933 .
Case(
"{ra}", RISCV::X1)
20934 .
Case(
"{sp}", RISCV::X2)
20935 .
Case(
"{gp}", RISCV::X3)
20936 .
Case(
"{tp}", RISCV::X4)
20937 .
Case(
"{t0}", RISCV::X5)
20938 .
Case(
"{t1}", RISCV::X6)
20939 .
Case(
"{t2}", RISCV::X7)
20940 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20941 .
Case(
"{s1}", RISCV::X9)
20942 .
Case(
"{a0}", RISCV::X10)
20943 .
Case(
"{a1}", RISCV::X11)
20944 .
Case(
"{a2}", RISCV::X12)
20945 .
Case(
"{a3}", RISCV::X13)
20946 .
Case(
"{a4}", RISCV::X14)
20947 .
Case(
"{a5}", RISCV::X15)
20948 .
Case(
"{a6}", RISCV::X16)
20949 .
Case(
"{a7}", RISCV::X17)
20950 .
Case(
"{s2}", RISCV::X18)
20951 .
Case(
"{s3}", RISCV::X19)
20952 .
Case(
"{s4}", RISCV::X20)
20953 .
Case(
"{s5}", RISCV::X21)
20954 .
Case(
"{s6}", RISCV::X22)
20955 .
Case(
"{s7}", RISCV::X23)
20956 .
Case(
"{s8}", RISCV::X24)
20957 .
Case(
"{s9}", RISCV::X25)
20958 .
Case(
"{s10}", RISCV::X26)
20959 .
Case(
"{s11}", RISCV::X27)
20960 .
Case(
"{t3}", RISCV::X28)
20961 .
Case(
"{t4}", RISCV::X29)
20962 .
Case(
"{t5}", RISCV::X30)
20963 .
Case(
"{t6}", RISCV::X31)
20965 if (XRegFromAlias != RISCV::NoRegister)
20966 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20975 if (Subtarget.hasStdExtF()) {
20977 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20978 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20979 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20980 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20981 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20982 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20983 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20984 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20985 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20986 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20987 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20988 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20989 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20990 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20991 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20992 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20993 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20994 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20995 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20996 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20997 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20998 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20999 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
21000 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
21001 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
21002 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
21003 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
21004 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
21005 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
21006 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
21007 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
21008 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
21010 if (FReg != RISCV::NoRegister) {
21011 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
21012 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
21013 unsigned RegNo = FReg - RISCV::F0_F;
21014 unsigned DReg = RISCV::F0_D + RegNo;
21015 return std::make_pair(DReg, &RISCV::FPR64RegClass);
21017 if (VT == MVT::f32 || VT == MVT::Other)
21018 return std::make_pair(FReg, &RISCV::FPR32RegClass);
21019 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
21020 unsigned RegNo = FReg - RISCV::F0_F;
21021 unsigned HReg = RISCV::F0_H + RegNo;
21022 return std::make_pair(HReg, &RISCV::FPR16RegClass);
21029 .
Case(
"{v0}", RISCV::V0)
21030 .
Case(
"{v1}", RISCV::V1)
21031 .
Case(
"{v2}", RISCV::V2)
21032 .
Case(
"{v3}", RISCV::V3)
21033 .
Case(
"{v4}", RISCV::V4)
21034 .
Case(
"{v5}", RISCV::V5)
21035 .
Case(
"{v6}", RISCV::V6)
21036 .
Case(
"{v7}", RISCV::V7)
21037 .
Case(
"{v8}", RISCV::V8)
21038 .
Case(
"{v9}", RISCV::V9)
21039 .
Case(
"{v10}", RISCV::V10)
21040 .
Case(
"{v11}", RISCV::V11)
21041 .
Case(
"{v12}", RISCV::V12)
21042 .
Case(
"{v13}", RISCV::V13)
21043 .
Case(
"{v14}", RISCV::V14)
21044 .
Case(
"{v15}", RISCV::V15)
21045 .
Case(
"{v16}", RISCV::V16)
21046 .
Case(
"{v17}", RISCV::V17)
21047 .
Case(
"{v18}", RISCV::V18)
21048 .
Case(
"{v19}", RISCV::V19)
21049 .
Case(
"{v20}", RISCV::V20)
21050 .
Case(
"{v21}", RISCV::V21)
21051 .
Case(
"{v22}", RISCV::V22)
21052 .
Case(
"{v23}", RISCV::V23)
21053 .
Case(
"{v24}", RISCV::V24)
21054 .
Case(
"{v25}", RISCV::V25)
21055 .
Case(
"{v26}", RISCV::V26)
21056 .
Case(
"{v27}", RISCV::V27)
21057 .
Case(
"{v28}", RISCV::V28)
21058 .
Case(
"{v29}", RISCV::V29)
21059 .
Case(
"{v30}", RISCV::V30)
21060 .
Case(
"{v31}", RISCV::V31)
21062 if (VReg != RISCV::NoRegister) {
21063 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
21064 return std::make_pair(VReg, &RISCV::VMRegClass);
21065 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
21066 return std::make_pair(VReg, &RISCV::VRRegClass);
21067 for (
const auto *RC :
21068 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
21069 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
21070 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
21071 return std::make_pair(VReg, RC);
21077 std::pair<Register, const TargetRegisterClass *> Res =
21083 if (Res.second == &RISCV::GPRF16RegClass ||
21084 Res.second == &RISCV::GPRF32RegClass ||
21085 Res.second == &RISCV::GPRPairRegClass)
21086 return std::make_pair(Res.first, &RISCV::GPRRegClass);
21094 if (ConstraintCode.
size() == 1) {
21095 switch (ConstraintCode[0]) {
21110 if (Constraint.
size() == 1) {
21111 switch (Constraint[0]) {
21114 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
21116 if (isInt<12>(CVal))
21129 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
21131 if (isUInt<5>(CVal))
21149 if (Subtarget.hasStdExtZtso()) {
21165 if (Subtarget.hasStdExtZtso()) {
21173 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Inst) &&
21192 if (Subtarget.hasForcedAtomics())
21197 if (Subtarget.hasStdExtZacas() &&
21198 (
Size >= 32 || Subtarget.hasStdExtZabha()))
21204 if (
Size < 32 && !Subtarget.hasStdExtZabha())
21217 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
21219 return Intrinsic::riscv_masked_atomicrmw_add_i32;
21221 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
21223 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
21225 return Intrinsic::riscv_masked_atomicrmw_max_i32;
21227 return Intrinsic::riscv_masked_atomicrmw_min_i32;
21229 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
21231 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
21240 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
21242 return Intrinsic::riscv_masked_atomicrmw_add_i64;
21244 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
21246 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
21248 return Intrinsic::riscv_masked_atomicrmw_max_i64;
21250 return Intrinsic::riscv_masked_atomicrmw_min_i64;
21252 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
21254 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
21280 unsigned XLen = Subtarget.
getXLen();
21304 unsigned ValWidth =
21309 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
21312 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
21324 if (Subtarget.hasForcedAtomics())
21328 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
21337 unsigned XLen = Subtarget.
getXLen();
21339 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
21344 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
21348 CmpXchgIntrID, Tys, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
21355 EVT DataVT)
const {
21371 return Subtarget.hasStdExtZfhmin();
21373 return Subtarget.hasStdExtF();
21375 return Subtarget.hasStdExtD();
21407 "RVVBitsPerBlock changed, audit needed");
21416 if (!Subtarget.hasVendorXTHeadMemIdx())
21422 Base =
Op->getOperand(0);
21424 int64_t RHSC =
RHS->getSExtValue();
21430 bool isLegalIndexedOffset =
false;
21431 for (
unsigned i = 0; i < 4; i++)
21432 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
21433 isLegalIndexedOffset =
true;
21437 if (!isLegalIndexedOffset)
21454 VT = LD->getMemoryVT();
21455 Ptr = LD->getBasePtr();
21456 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21457 VT = ST->getMemoryVT();
21458 Ptr = ST->getBasePtr();
21474 if (Subtarget.hasVendorXCVmem() && !Subtarget.
is64Bit()) {
21479 Base = LS->getBasePtr();
21483 if (
Base ==
Op->getOperand(0))
21485 else if (
Base ==
Op->getOperand(1))
21497 VT = LD->getMemoryVT();
21498 Ptr = LD->getBasePtr();
21499 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21500 VT = ST->getMemoryVT();
21501 Ptr = ST->getBasePtr();
21544 const Constant *PersonalityFn)
const {
21549 const Constant *PersonalityFn)
const {
21564 bool IsSigned)
const {
21579 const bool HasZmmul = Subtarget.hasStdExtZmmul();
21583 auto *ConstNode = cast<ConstantSDNode>(
C);
21584 const APInt &Imm = ConstNode->getAPIntValue();
21587 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
21588 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
21592 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
21593 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
21594 (Imm - 8).isPowerOf2()))
21599 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
21600 ConstNode->hasOneUse()) {
21601 APInt ImmS = Imm.ashr(Imm.countr_zero());
21602 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
21603 (1 - ImmS).isPowerOf2())
21626 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
21635 unsigned *
Fast)
const {
21638 *
Fast = Subtarget.enableUnalignedScalarMem();
21639 return Subtarget.enableUnalignedScalarMem();
21655 *
Fast = Subtarget.enableUnalignedVectorMem();
21656 return Subtarget.enableUnalignedVectorMem();
21665 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
21677 if (
Op.size() < MinVLenInBytes)
21692 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
21696 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
21698 if (
Op.isFixedDstAlign())
21699 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
21701 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
21709 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
21710 bool IsABIRegCopy =
CC.has_value();
21713 MVT PairVT = Subtarget.
is64Bit() ? MVT::i128 : MVT::i64;
21714 if ((ValueVT == PairVT ||
21715 (!Subtarget.
is64Bit() && Subtarget.hasStdExtZdinx() &&
21716 ValueVT == MVT::f64)) &&
21717 NumParts == 1 && PartVT == MVT::Untyped) {
21720 if (ValueVT == MVT::f64)
21729 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21730 PartVT == MVT::f32) {
21745 [[maybe_unused]]
unsigned ValLMUL =
21749 [[maybe_unused]]
unsigned PartLMUL =
21752 assert(ValNF == PartNF && ValLMUL == PartLMUL &&
21753 "RISC-V vector tuple type only accepts same register class type "
21769 if (PartVTBitSize % ValueVTBitSize == 0) {
21770 assert(PartVTBitSize >= ValueVTBitSize);
21777 if (ValueEltVT != PartEltVT) {
21778 if (PartVTBitSize > ValueVTBitSize) {
21780 assert(Count != 0 &&
"The number of element should not be zero.");
21781 EVT SameEltTypeVT =
21803 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
21804 bool IsABIRegCopy =
CC.has_value();
21806 MVT PairVT = Subtarget.
is64Bit() ? MVT::i128 : MVT::i64;
21807 if ((ValueVT == PairVT ||
21808 (!Subtarget.
is64Bit() && Subtarget.hasStdExtZdinx() &&
21809 ValueVT == MVT::f64)) &&
21810 NumParts == 1 && PartVT == MVT::Untyped) {
21821 if (ValueVT == MVT::f64)
21826 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21827 PartVT == MVT::f32) {
21844 if (PartVTBitSize % ValueVTBitSize == 0) {
21845 assert(PartVTBitSize >= ValueVTBitSize);
21846 EVT SameEltTypeVT = ValueVT;
21853 if (ValueEltVT != PartEltVT) {
21855 assert(Count != 0 &&
"The number of element should not be zero.");
21872 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21879 unsigned Opc =
N->getOpcode();
21907 if (M->getStackProtectorGuard() ==
"tls") {
21909 int Offset = M->getStackProtectorGuardOffset();
21917 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21931 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21936 if (FVTy->getNumElements() < 2)
21951 return Factor * LMUL <= 8;
21955 Align Alignment)
const {
21967 if (!Subtarget.enableUnalignedVectorMem() &&
21975 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21976 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21977 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21978 Intrinsic::riscv_seg8_load};
21999 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
22011 unsigned ScalarSizeInBytes = VTy->getScalarSizeInBits() / 8;
22012 Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);
22013 Value *
Offset = ConstantInt::get(XLenTy, Indices[0] * ScalarSizeInBytes);
22020 {VTy, BasePtr->getType(), Stride->
getType()},
22021 {BasePtr, Stride, Mask, VL});
22024 Shuffles[0]->replaceAllUsesWith(CI);
22028 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
22034 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
22036 Shuffles[i]->replaceAllUsesWith(SubVec);
22043 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
22044 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
22045 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
22046 Intrinsic::riscv_seg8_store};
22066 unsigned Factor)
const {
22069 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
22072 ShuffleVTy->getNumElements() / Factor);
22074 SI->getPointerAddressSpace(),
22075 SI->getDataLayout()))
22086 unsigned ScalarSizeInBytes = ShuffleVTy->getScalarSizeInBits() / 8;
22088 auto *DataVTy = cast<FixedVectorType>(
Data->getType());
22089 Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);
22090 Value *
Offset = ConstantInt::get(XLenTy, Index * ScalarSizeInBytes);
22096 Intrinsic::experimental_vp_strided_store,
22097 {
Data->getType(), BasePtr->getType(), Stride->
getType()},
22098 {
Data, BasePtr, Stride, Mask, VL});
22107 {VTy, SI->getPointerOperandType(), XLenTy});
22111 for (
unsigned i = 0; i < Factor; i++) {
22120 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
22121 Ops.
append({SI->getPointerOperand(), VL});
22138 const unsigned Factor = 2;
22150 if (
auto *FVTy = dyn_cast<FixedVectorType>(ResVTy)) {
22151 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
22158 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
22159 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
22160 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
22161 Intrinsic::riscv_vlseg8};
22168 NumElts * SEW / 8),
22174 IntrIds[Factor - 2], {VecTupTy, XLenTy},
22176 ConstantInt::get(XLenTy,
Log2_64(SEW))});
22180 for (
unsigned i = 0; i < Factor; ++i) {
22182 Intrinsic::riscv_tuple_extract, {ResVTy, VecTupTy},
22200 if (
II->getIntrinsicID() != Intrinsic::vector_interleave2)
22203 const unsigned Factor = 2;
22205 VectorType *InVTy = cast<VectorType>(
II->getArgOperand(0)->getType());
22209 SI->getPointerAddressSpace(),
DL))
22214 if (
auto *FVTy = dyn_cast<FixedVectorType>(InVTy)) {
22215 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
22217 {InVTy, SI->getPointerOperandType(), XLenTy},
22218 {
II->getArgOperand(0),
II->getArgOperand(1),
22219 SI->getPointerOperand(), VL});
22222 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
22223 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
22224 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
22225 Intrinsic::riscv_vsseg8};
22230 SI->getContext(),
"riscv.vector.tuple",
22232 NumElts * SEW / 8),
22236 SI->getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
22241 for (
unsigned i = 0; i < Factor; ++i)
22243 Intrinsic::riscv_tuple_insert, {VecTupTy, InVTy},
22244 {StoredVal, II->getArgOperand(i), Builder.getInt32(i)});
22246 Builder.
CreateCall(VssegNFunc, {StoredVal, SI->getPointerOperand(), VL,
22247 ConstantInt::get(XLenTy,
Log2_64(SEW))});
22258 "Invalid call instruction for a KCFI check");
22260 MBBI->getOpcode()));
22263 Target.setIsRenamable(
false);
22271#define GET_REGISTER_MATCHER
22272#include "RISCVGenAsmMatcher.inc"
22278 if (Reg == RISCV::NoRegister)
22280 if (Reg == RISCV::NoRegister)
22292 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
22294 if (NontemporalInfo ==
nullptr)
22302 int NontemporalLevel = 5;
22303 const MDNode *RISCVNontemporalInfo =
22304 I.getMetadata(
"riscv-nontemporal-domain");
22305 if (RISCVNontemporalInfo !=
nullptr)
22308 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
22312 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
22313 "RISC-V target doesn't support this non-temporal domain.");
22315 NontemporalLevel -= 2;
22317 if (NontemporalLevel & 0b1)
22319 if (NontemporalLevel & 0b10)
22332 return TargetFlags;
22342 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
22345 return Subtarget.hasStdExtZbb() &&
22356 if (Subtarget.hasStdExtZalasr()) {
22357 if (Subtarget.hasStdExtZtso()) {
22362 auto *LI = dyn_cast<LoadInst>(
I);
22363 auto *SI = dyn_cast<StoreInst>(
I);
22373 return isa<LoadInst>(
I) || isa<StoreInst>(
I);
22380 return isa<LoadInst>(
I) || isa<StoreInst>(
I);
22387 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
22388 Op == Instruction::And ||
Op == Instruction::Or ||
22389 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
22390 Op == Instruction::ShuffleVector ||
Op == Instruction::Load ||
22391 Op == Instruction::Freeze ||
Op == Instruction::Store)
22399 !isa<ReturnInst>(&Inst))
22402 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
22403 if (AI->getAllocatedType()->isScalableTy())
22411RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
22419 if (!Subtarget.hasShortForwardBranchOpt())
22421 EVT VT =
N->getValueType(0);
22422 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
22426 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
22431bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
22432 EVT VT,
const APInt &AndMask)
const {
22433 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
22434 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
22438unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
22446 if (Subtarget.hasStdExtZicfilp()) {
22470std::pair<const TargetRegisterClass *, uint8_t>
22490#define GET_RISCVVIntrinsicsTable_IMPL
22491#include "RISCVGenSearchableTables.inc"
22506 Align StackAlign)
const {
22510 unsigned StackProbeSize =
22514 return StackProbeSize ? StackProbeSize : StackAlign.
value();
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static StringRef getName(Value *V)
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static bool hasPassthruOp(unsigned Opcode)
Return true if a RISC-V target specified op has a passthru operand.
static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< APInt > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static unsigned getPACKOpcode(unsigned DestBW, const RISCVSubtarget &Subtarget)
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineOp_VLToVWOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary or FMA operation to its equivalent VW or VW_W form.
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Given a shuffle where the indices are disjoint between the two sources, e.g.:
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG)
static SDValue combineScalarCTPOPToVCPOP(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isSpreadMask(ArrayRef< int > Mask, unsigned Factor, unsigned &Index)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isPromotedOpNeedingSplit(SDValue Op, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG)
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT, SDValue Src, unsigned Factor, unsigned Index, SelectionDAG &DAG)
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG)
static SDValue getSingleShuffleSrc(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue performVFMADD_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG)
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index, const SDLoc &DL, SelectionDAG &DAG)
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Double the element size of the build vector to reduce the number of vslide1down in the build vector c...
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static bool isCompressMask(ArrayRef< int > Mask)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
static unsigned getRISCVWOpcode(unsigned Opcode)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
void clearAllBits()
Set every bit to 0.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * getAllOnesMask(ElementCount NumElts)
Return an all true boolean vector (mask) with NumElts lanes.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Type * getPointerOperandType() const
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
Instances of this class represent a single low-level machine instruction.
MCContext & getContext() const
Generic base class for all target subtargets.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
bool isRISCVVectorTuple() const
Return true if this is a RISCV vector tuple type where the runtime length is machine dependent.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
static MVT getRISCVVectorTupleVT(unsigned Sz, unsigned NFields)
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
unsigned getRISCVVectorTupleNumFields() const
Given a RISC-V vector tuple type, return the num_fields.
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A RISCV-specific constant pool value.
static RISCVConstantPoolValue * Create(const GlobalValue *GV)
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
unsigned getMaxStoresPerMemcpy(bool OptSize) const
bool hasStdExtDOrZdinx() const
unsigned getMaxLoadsPerMemcmp(bool OptSize) const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
unsigned getMaxStoresPerMemset(bool OptSize) const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
bool hasVInstructionsBF16Minimal() const
unsigned getDLenFactor() const
unsigned getMaxStoresPerMemmove(bool OptSize) const
bool hasVInstructionsF16Minimal() const
unsigned getMaxGluedStoresPerMemcpy() const
bool hasConditionalMoveFusion() const
bool hasVInstructionsF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
bool isRegisterReservedByUser(Register i) const override
std::optional< unsigned > getRealVLen() const
bool hasOptimizedSegmentLoadStore(unsigned NF) const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool isTargetAndroid() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI, SmallVectorImpl< Instruction * > &DeadInsts) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI, SmallVectorImpl< Instruction * > &DeadInsts) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this functions.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
static RISCVII::VLMUL getLMUL(MVT VT)
int getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const override
Return the number of registers for a given MVT, for inline assembly.
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
iterator_range< user_iterator > users()
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
static ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getRegister(Register Reg, EVT VT)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isDeInterleaveMaskOfFactor(ArrayRef< int > Mask, unsigned Factor, unsigned &Index)
Check if the mask is a DE-interleave mask of the given factor Factor like: <Index,...
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
static TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool EnableExtLdPromotion
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
const MCSubtargetInfo * getMCSubtargetInfo() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual TargetLoweringObjectFile * getObjFileLowering() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool isRegisterReservedByUser(Register R) const
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
bool isTargetExtTy() const
Return true if this is a target extension type.
bool isScalableTy(SmallPtrSetImpl< const Type * > &Visited) const
Return true if this is a type whose size is a known multiple of vscale.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
unsigned getOperandNo() const
Return the operand # of this use in its User.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
Type * getElementType() const
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
@ TRUNCATE_SSAT_S
TRUNCATE_[SU]SAT_[SU] - Truncate for saturated operand [SU] located in middle, prefix for SAT means i...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SplitF64
Turns a f64 into a pair of i32s.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ TRUNCATE_VECTOR_VL_USAT
@ LAST_RISCV_STRICTFP_OPCODE
@ BuildPairF64
Turns a pair of i32s into an f64.
@ BuildGPRPair
Turn a pair of i<xlen>s into an even-odd register pair (untyped).
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ SplitGPRPair
Turn an even-odd register pair (untyped) into a pair of i<xlen>s.
@ TRUNCATE_VECTOR_VL_SSAT
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost, bool FreeZeroes)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
uint32_t read32le(const void *P)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
MCCodeEmitter * createRISCVMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx)
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool RISCVCCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
RISCVCCAssignFn - This target-specific function extends the default CCValAssign with additional infor...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getRISCVVectorTupleNumFields() const
Given a RISCV vector tuple type, return the num_fields.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)